Skip add_host_metadata for forwarded event logs (elastic#18153)

Update config examples to use the "forwarded" tag to skip adding host metadata.

Also disable host.name being added by libbeat. This field was overwritten by
the winlog.computer_name so it didn't serve any purpose to have libbeat set it.

Relates elastic#13920

(cherry picked from commit f80f82c)

CHANGELOG.next.asciidoc

@@ -269,6 +269,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Kerberos support to Elasticsearch output. {pull}17927[17927]
 - Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000]
 - Add keystore support for autodiscover static configurations. {pull]16306[16306]
+- Add config example of how to skip the `add_host_metadata` processor when forwarding logs. {issue}13920[13920] {pull}18153[18153]
 
 *Auditbeat*
 

dev-tools/mage/config.go

@@ -116,6 +116,7 @@ func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error
 		"UseDockerMetadataProcessor":     true,
 		"UseKubernetesMetadataProcessor": false,
 		"ExcludeDashboards":              false,
+		"UseProcessorsTemplate":          false,
 	}
 	for k, v := range args.ExtraVars {
 		params[k] = v

libbeat/_meta/config.yml.tmpl

@@ -90,6 +90,7 @@ output.elasticsearch:
   #ssl.key: "/etc/pki/client/cert.key"
 {{end}}
 #================================ Processors =====================================
+{{if .UseProcessorsTemplate}}{{template "processors.yml.tmpl" .}}{{else -}}
 {{if not .UseObserverProcessor}}
 # Configure processors to enhance or manipulate events generated by the beat.
 
@@ -112,7 +113,7 @@ processors:
         #name: us-east-1a
         # Lat, Lon "
         #location: "37.926868, -78.024902"
-{{end}}
+{{end}}{{end}}
 #================================ Logging =====================================
 
 # Sets log level. The default log level is info.

winlogbeat/_meta/beat.yml.tmpl

@@ -2,18 +2,11 @@
 winlogbeat.event_logs:
   - name: Application
     ignore_older: 72h
-{{if .Reference}}
-    # Set to true to publish fields with null values in events.
-    #keep_null: false
-{{end}}
+
   - name: System
-{{if .Reference}}
-    # Set to true to publish fields with null values in events.
-    #keep_null: false
-{{end}}
+
   - name: Security
-{{if .Reference}}
-    # Set to true to publish fields with null values in events.
-    #keep_null: false
-{{end}}
+
+  - name: ForwardedEvents
+    tags: [forwarded]
 {{if not .Reference}}{{ template "elasticsearch_settings" . }}{{end}}

winlogbeat/_meta/common.yml.tmpl

@@ -34,3 +34,9 @@ setup.template.settings:
   #index.codec: best_compression
   #_source.enabled: false
 {{end -}}
+{{define "processors.yml.tmpl"}}
+processors:
+  - add_host_metadata:
+      when.not.contains.tags: forwarded
+  - add_cloud_metadata: ~
+{{end -}}

winlogbeat/cmd/root.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"github.com/elastic/beats/v7/libbeat/cmd"
 	"github.com/elastic/beats/v7/libbeat/cmd/instance"
+	"github.com/elastic/beats/v7/libbeat/publisher/processing"
 	"github.com/elastic/beats/v7/winlogbeat/beater"
 
 	// Register fields.
@@ -35,4 +36,8 @@ import (
 var Name = "winlogbeat"
 
 // RootCmd to handle beats cli
-var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{Name: Name, HasDashboards: true})
+var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{
+	Name:          Name,
+	HasDashboards: true,
+	Processing:    processing.MakeDefaultSupport(true, processing.WithECS, processing.WithAgentMeta()),
+})

winlogbeat/scripts/mage/config.go

@@ -54,7 +54,8 @@ func configFileParams() devtools.ConfigFileParams {
 			devtools.LibbeatDir("_meta/config.docker.yml"),
 		},
 		ExtraVars: map[string]interface{}{
-			"GOOS": "windows",
+			"GOOS":                  "windows",
+			"UseProcessorsTemplate": true,
 		},
 	}
 }

winlogbeat/winlogbeat.reference.yml

@@ -26,19 +26,12 @@ winlogbeat.event_logs:
   - name: Application
     ignore_older: 72h
 
-    # Set to true to publish fields with null values in events.
-    #keep_null: false
-
   - name: System
 
-    # Set to true to publish fields with null values in events.
-    #keep_null: false
-
   - name: Security
 
-    # Set to true to publish fields with null values in events.
-    #keep_null: false
-
+  - name: ForwardedEvents
+    tags: [forwarded]
 
 
 #================================ General ======================================

winlogbeat/winlogbeat.yml

@@ -25,6 +25,8 @@ winlogbeat.event_logs:
 
   - name: Security
 
+  - name: ForwardedEvents
+    tags: [forwarded]
 #==================== Elasticsearch template settings ==========================
 
 setup.template.settings:
@@ -125,12 +127,10 @@ output.elasticsearch:
 
 #================================ Processors =====================================
 
-# Configure processors to enhance or manipulate events generated by the beat.
-
 processors:
-  - add_host_metadata: ~
+  - add_host_metadata:
+      when.not.contains.tags: forwarded
   - add_cloud_metadata: ~
-  - add_docker_metadata: ~
 
 #================================ Logging =====================================
 

x-pack/winlogbeat/_meta/beat.yml.tmpl

@@ -19,4 +19,18 @@ winlogbeat.event_logs:
           id: sysmon
           file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js
 
+  - name: ForwardedEvents
+    tags: [forwarded]
+    processors:
+      - script:
+          when.equals.winlog.channel: Security
+          lang: javascript
+          id: security
+          file: ${path.home}/module/security/config/winlogbeat-security.js
+      - script:
+          when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational
+          lang: javascript
+          id: sysmon
+          file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js
+
 {{if not .Reference}}{{ template "elasticsearch_settings" . }}{{end}}

x-pack/winlogbeat/winlogbeat.reference.yml

@@ -42,6 +42,20 @@ winlogbeat.event_logs:
           id: sysmon
           file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js
 
+  - name: ForwardedEvents
+    tags: [forwarded]
+    processors:
+      - script:
+          when.equals.winlog.channel: Security
+          lang: javascript
+          id: security
+          file: ${path.home}/module/security/config/winlogbeat-security.js
+      - script:
+          when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational
+          lang: javascript
+          id: sysmon
+          file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js
+
 
 
 #================================ General ======================================

x-pack/winlogbeat/winlogbeat.yml

@@ -37,6 +37,20 @@ winlogbeat.event_logs:
           id: sysmon
           file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js
 
+  - name: ForwardedEvents
+    tags: [forwarded]
+    processors:
+      - script:
+          when.equals.winlog.channel: Security
+          lang: javascript
+          id: security
+          file: ${path.home}/module/security/config/winlogbeat-security.js
+      - script:
+          when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational
+          lang: javascript
+          id: sysmon
+          file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js
+
 #==================== Elasticsearch template settings ==========================
 
 setup.template.settings:
@@ -137,12 +151,10 @@ output.elasticsearch:
 
 #================================ Processors =====================================
 
-# Configure processors to enhance or manipulate events generated by the beat.
-
 processors:
-  - add_host_metadata: ~
+  - add_host_metadata:
+      when.not.contains.tags: forwarded
   - add_cloud_metadata: ~
-  - add_docker_metadata: ~
 
 #================================ Logging =====================================