Merge remote-tracking branch 'origin/master' into make-notice-check-n…

…o-changes

.ci/packaging.groovy

@@ -208,7 +208,8 @@ pipeline {
                   'packetbeat',
                   'x-pack/auditbeat',
                   'x-pack/dockerlogbeat',
-                  'x-pack/elastic-agent',
+                  // See https://github.com/elastic/beats/issues/26239
+                  // 'x-pack/elastic-agent',
                   'x-pack/filebeat',
                   'x-pack/heartbeat',
                   'x-pack/metricbeat',

.go-version

@@ -1 +1 @@
-1.16.4
+1.16.5

CHANGELOG-developer.next.asciidoc

@@ -116,3 +116,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.15.12. {pull}25629[25629]
 - Update Go version to 1.16.4. {issue}25346[25346] {pull}25671[25671]
 - Add sorting to array fields for generated data files (*-generated.json) {pull}25320[25320]
+- Update Go version to 1.16.5. {issue}26182[26182] {pull}26186[26186]

CHANGELOG.next.asciidoc

@@ -242,6 +242,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix handling of `file_selectors` in aws-s3 input. {pull}25792[25792]
 - Fix ILM alias creation when write alias exists and initial index does not exist {pull}26143[26143]
 - Include date separator in the filename prefix of `dateRotator` to make sure nothing gets purged accidentally {pull}26176[26176]
+- In the script processor, the `decode_xml` and `decode_xml_wineventlog` processors are now available as `DecodeXML` and `DecodeXMLWineventlog` respectively.
 
 *Auditbeat*
 
@@ -280,6 +281,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523]
 - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
 - Fix issue with m365_defender, when parsing incidents that has no alerts attached: {pull}25421[25421]
+- Fix default config template values for paths on oracle module: {pull}26276[26276]
+- Fix bug in aws-s3 input where the end of gzipped log files might have been discarded. {pull}26260[26260]
 
 *Filebeat*
 
@@ -496,6 +499,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
 - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
 - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
+- Major refactor of system/cpu and system/core metrics. {pull}25771[25771]
 
 *Packetbeat*
 
@@ -821,6 +825,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Make `filestream` input GA. {pull}26127[26127]
 - Add new `parser` to `filestream` input: `container`. {pull}26115[26115]
 - Add support for ISO8601 timestamps in Zeek fileset {pull}25564[25564]
+- Add `preserve_original_event` option to `o365audit` input. {pull}26273[26273]
+- Add `log.flags` to events created by the `aws-s3` input. {pull}26267[26267]
+- Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267]
 
 *Heartbeat*
 
@@ -955,6 +962,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Reduce number of requests done by kubernetes metricsets to kubelet. {pull}25782[25782]
 - Migrate rds metricsets to use cloudwatch input. {pull}26077[26077]
 - Migrate sqs metricsets to use cloudwatch input. {pull}26117[26117]
+- Add total CPU to vSphere virtual machine metrics. {pull}26167[26167]
 
 *Packetbeat*
 

Vagrantfile

@@ -307,6 +307,7 @@ Vagrant.configure("2") do |config|
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: $freebsdShellUpdate, privileged: true
     c.vm.provision "shell", inline: gvmProvision(arch="amd64", os="freebsd"), privileged: false
+    c.vm.provision "shell", inline: "sudo mount -t linprocfs /dev/null /proc", privileged: false
   end
 
   # OpenBSD 6.0

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.4
+FROM golang:1.16.5
 
 RUN \
     apt-get update \

filebeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.4
+FROM golang:1.16.5
 
 RUN \
     apt-get update \

filebeat/docs/fields.asciidoc

@@ -125697,7 +125697,7 @@ S3 fields from s3 input.
 
 
 
-*`bucket_name`*::
+*`bucket.name`*::
 +
 --
 Name of the S3 bucket that this log retrieved from.
@@ -125707,7 +125707,17 @@ type: keyword
 
 --
 
-*`object_key`*::
+*`bucket.arn`*::
++
+--
+ARN of the S3 bucket that this log retrieved from.
+
+
+type: keyword
+
+--
+
+*`object.key`*::
 +
 --
 Name of the S3 object that this log retrieved from.
@@ -125717,6 +125727,15 @@ type: keyword
 
 --
 
+*`metadata`*::
++
+--
+AWS S3 object metadata values.
+
+type: flattened
+
+--
+
 [[exported-fields-santa]]
 == Google Santa fields
 

filebeat/docs/modules/iis.asciidoc

@@ -11,6 +11,11 @@ This file is generated! See scripts/docs_collector.py
 The +{modulename}+ module parses access and error logs created by the
 Internet Information Services (IIS) HTTP server.
 
+[IMPORTANT]
+=====
+The +{modulename}+ module currently supports only the default W3C log format.
+=====
+
 include::../include/what-happens.asciidoc[]
 
 include::../include/gs-link.asciidoc[]

filebeat/module/iis/_meta/docs.asciidoc

@@ -6,6 +6,11 @@
 The +{modulename}+ module parses access and error logs created by the
 Internet Information Services (IIS) HTTP server.
 
+[IMPORTANT]
+=====
+The +{modulename}+ module currently supports only the default W3C log format.
+=====
+
 include::../include/what-happens.asciidoc[]
 
 include::../include/gs-link.asciidoc[]

generator/common/Makefile

@@ -23,6 +23,7 @@ test-package: test
 	cd ${BEAT_PATH} ; \
 	export PATH=$${GOPATH}/bin:$${PATH}; \
 	go mod tidy && \
+	go mod download all && \
 	mage package
 
 .PHONY: prepare-test

heartbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.4
+FROM golang:1.16.5
 
 RUN \
     apt-get update \

journalbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.4
+FROM golang:1.16.5
 
 RUN \
     apt-get update \

libbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.4
+FROM golang:1.16.5
 
 RUN \
     apt-get update \

libbeat/docs/version.asciidoc

@@ -1,6 +1,6 @@
 :stack-version: 8.0.0
 :doc-branch: master
-:go-version: 1.16.4
+:go-version: 1.16.5
 :release-state: unreleased
 :python: 3.7
 :docker: 1.12

libbeat/logp/config.go

@@ -80,3 +80,14 @@ func DefaultConfig(environment Environment) Config {
 		addCaller:   true,
 	}
 }
+
+// LogFilename returns the base filename to which logs will be written for
+// the "files" log output. If another log output is used, or `logging.files.name`
+// is unspecified, then the beat name will be returned.
+func (cfg Config) LogFilename() string {
+	name := cfg.Beat
+	if cfg.Files.Name != "" {
+		name = cfg.Files.Name
+	}
+	return name
+}

libbeat/logp/core.go

@@ -196,6 +196,13 @@ func makeOptions(cfg Config) []zap.Option {
 	if cfg.development {
 		options = append(options, zap.Development())
 	}
+	if cfg.ECSEnabled {
+		fields := []zap.Field{
+			zap.String("service.name", cfg.Beat),
+			zap.String("event.dataset", cfg.LogFilename()),
+		}
+		options = append(options, zap.Fields(fields...))
+	}
 	return options
 }
 
@@ -226,11 +233,7 @@ func makeEventLogOutput(cfg Config) (zapcore.Core, error) {
 }
 
 func makeFileOutput(cfg Config) (zapcore.Core, error) {
-	name := cfg.Beat
-	if cfg.Files.Name != "" {
-		name = cfg.Files.Name
-	}
-	filename := paths.Resolve(paths.Logs, filepath.Join(cfg.Files.Path, name))
+	filename := paths.Resolve(paths.Logs, filepath.Join(cfg.Files.Path, cfg.LogFilename()))
 
 	rotator, err := file.NewFileRotator(filename,
 		file.MaxSizeBytes(cfg.Files.MaxSize),

libbeat/logp/core_test.go

@@ -146,3 +146,30 @@ func TestNotDebugAllStdoutDisablesDefaultGoLogger(t *testing.T) {
 	DevelopmentSetup(WithSelectors("other"), WithLevel(InfoLevel))
 	assert.Equal(t, ioutil.Discard, golog.Writer())
 }
+
+func TestLoggingECSFields(t *testing.T) {
+	cfg := Config{
+		Beat:        "beat1",
+		Level:       DebugLevel,
+		development: true,
+		ECSEnabled:  true,
+		Files: FileConfig{
+			Name: "beat1.log",
+		},
+	}
+	ToObserverOutput()(&cfg)
+	Configure(cfg)
+
+	logger := NewLogger("tester")
+
+	logger.Debug("debug")
+	logs := ObserverLogs().TakeAll()
+	if assert.Len(t, logs, 1) {
+		if assert.Len(t, logs[0].Context, 2) {
+			assert.Equal(t, "service.name", logs[0].Context[0].Key)
+			assert.Equal(t, "beat1", logs[0].Context[0].String)
+			assert.Equal(t, "event.dataset", logs[0].Context[1].Key)
+			assert.Equal(t, "beat1.log", logs[0].Context[1].String)
+		}
+	}
+}