[Journalbeat][ecs] Journalbeat ecs 1.8 (elastic#23737)

* Improve ECS mappings and upgrade to ecs 1.8

* Run mage update

CHANGELOG.next.asciidoc

@@ -843,6 +843,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Journalbeat*
 
+- Update Journalbeat to ECS 1.8. {pull}23737[23737]
+
 *Metricbeat*
 
 - Move the windows pdh implementation from perfmon to a shared location in order for future modules/metricsets to make use of. {pull}15503[15503]

journalbeat/_meta/fields.common.yml

@@ -43,19 +43,19 @@
                   example: 3
                   description: >
                     The audit session of the object process.
-            - name: cmd
+            - name: process.command_line
               type: keyword
               required: false
               example: "/lib/systemd/systemd --user"
               description: >
                 The command line of the process.
-            - name: name
+            - name: process.name
               type: keyword
               required: false
               example: "/lib/systemd/systemd"
               description: >
                 Name of the executable.
-            - name: executable
+            - name: process.executable
               type: keyword
               required: false
               description: >
@@ -176,7 +176,7 @@
                   example: 3
                   description: >
                     The audit session of the source process.
-            - name: cmd
+            - name: command_line
               type: keyword
               required: false
               example: "/lib/systemd/systemd --user"

journalbeat/cmd/root.go

@@ -35,7 +35,7 @@ const (
 	Name = "journalbeat"
 
 	// ecsVersion specifies the version of ECS that Winlogbeat is implementing.
-	ecsVersion = "1.7.0"
+	ecsVersion = "1.8.0"
 )
 
 // withECSVersion is a modifier that adds ecs.version to events.

journalbeat/docs/fields.asciidoc

@@ -251,7 +251,7 @@ required: False
 
 --
 
-*`journald.object.cmd`*::
+*`journald.object.process.command_line`*::
 +
 --
 The command line of the process.
@@ -265,7 +265,7 @@ required: False
 
 --
 
-*`journald.object.name`*::
+*`journald.object.process.name`*::
 +
 --
 Name of the executable.
@@ -279,7 +279,7 @@ required: False
 
 --
 
-*`journald.object.executable`*::
+*`journald.object.process.executable`*::
 +
 --
 Path to the the executable.
@@ -542,7 +542,7 @@ required: False
 
 --
 
-*`journald.process.cmd`*::
+*`journald.process.command_line`*::
 +
 --
 The command line of the process.

journalbeat/pkg/journalfield/conv.go

@@ -19,6 +19,7 @@ package journalfield
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -84,7 +85,7 @@ func (c *Converter) Convert(entryFields map[string]string) common.MapStr {
 		fields.Put("journald.custom", custom)
 	}
 
-	return fields
+	return withECSEnrichment(fields)
 }
 
 func convertValue(fc Conversion, value string) (interface{}, error) {
@@ -106,6 +107,91 @@ func convertValue(fc Conversion, value string) (interface{}, error) {
 	return value, nil
 }
 
+func withECSEnrichment(fields common.MapStr) common.MapStr {
+	// from https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
+	// we see journald.object fields are populated by systemd on behalf of a different program
+	// so we want them to favor their use in root fields as they are the from the effective program
+	// performing the action.
+	setGidUidFields("journald", fields)
+	setGidUidFields("journald.object", fields)
+	setProcessFields("journald", fields)
+	setProcessFields("journald.object", fields)
+	return fields
+}
+
+func setGidUidFields(prefix string, fields common.MapStr) {
+	var auditLoginUid string
+	if found, _ := fields.HasKey(prefix + ".audit.login_uid"); found {
+		auditLoginUid = fmt.Sprint(getIntegerFromFields(prefix+".audit.login_uid", fields))
+		fields.Put("user.id", auditLoginUid)
+	}
+
+	if found, _ := fields.HasKey(prefix + ".uid"); !found {
+		return
+	}
+
+	uid := fmt.Sprint(getIntegerFromFields(prefix+".uid", fields))
+	gid := fmt.Sprint(getIntegerFromFields(prefix+".gid", fields))
+	if auditLoginUid != "" && auditLoginUid != uid {
+		putStringIfNotEmtpy("user.effective.id", uid, fields)
+		putStringIfNotEmtpy("user.effective.group.id", gid, fields)
+	} else {
+		putStringIfNotEmtpy("user.id", uid, fields)
+		putStringIfNotEmtpy("user.group.id", gid, fields)
+	}
+}
+
+var cmdlineRegexp = regexp.MustCompile(`"(\\"|[^"])*?"|[^\s]+`)
+
+func setProcessFields(prefix string, fields common.MapStr) {
+	if found, _ := fields.HasKey(prefix + ".pid"); found {
+		pid := getIntegerFromFields(prefix+".pid", fields)
+		fields.Put("process.pid", pid)
+	}
+
+	name := getStringFromFields(prefix+".name", fields)
+	if name != "" {
+		fields.Put("process.name", name)
+	}
+
+	executable := getStringFromFields(prefix+".executable", fields)
+	if executable != "" {
+		fields.Put("process.executable", executable)
+	}
+
+	cmdline := getStringFromFields(prefix+".process.command_line", fields)
+	if cmdline == "" {
+		return
+	}
+
+	fields.Put("process.command_line", cmdline)
+
+	args := cmdlineRegexp.FindAllString(cmdline, -1)
+	if len(args) > 0 {
+		fields.Put("process.args", args)
+		fields.Put("process.args_count", len(args))
+	}
+}
+
+func getStringFromFields(key string, fields common.MapStr) string {
+	value, _ := fields.GetValue(key)
+	str, _ := value.(string)
+	return str
+}
+
+func getIntegerFromFields(key string, fields common.MapStr) int64 {
+	value, _ := fields.GetValue(key)
+	i, _ := value.(int64)
+	return i
+}
+
+func putStringIfNotEmtpy(k, v string, fields common.MapStr) {
+	if v == "" {
+		return
+	}
+	fields.Put(k, v)
+}
+
 // helpers for creating a field conversion table.
 
 var ignoredField = Conversion{Dropped: true}

journalbeat/pkg/journalfield/conv_test.go

@@ -39,8 +39,10 @@ func TestConversion(t *testing.T) {
 				sdjournal.SD_JOURNAL_FIELD_BOOT_ID: "123456",
 			},
 			want: common.MapStr{
-				"host": common.MapStr{
-					"boot_id": "123456",
+				"journald": common.MapStr{
+					"host": common.MapStr{
+						"boot_id": "123456",
+					},
 				},
 			},
 		},

journalbeat/pkg/journalfield/default.go

@@ -28,9 +28,9 @@ var journaldEventFields = FieldConversion{
 	"COREDUMP_USER_UNIT":                         text("journald.coredump.user_unit"),
 	"OBJECT_AUDIT_LOGINUID":                      integer("journald.object.audit.login_uid"),
 	"OBJECT_AUDIT_SESSION":                       integer("journald.object.audit.session"),
-	"OBJECT_CMDLINE":                             text("journald.object.cmd"),
-	"OBJECT_COMM":                                text("journald.object.name"),
-	"OBJECT_EXE":                                 text("journald.object.executable"),
+	"OBJECT_CMDLINE":                             text("journald.object.process.command_line"),
+	"OBJECT_COMM":                                text("journald.object.process.name"),
+	"OBJECT_EXE":                                 text("journald.object.process.executable"),
 	"OBJECT_GID":                                 integer("journald.object.gid"),
 	"OBJECT_PID":                                 integer("journald.object.pid"),
 	"OBJECT_SYSTEMD_OWNER_UID":                   integer("journald.object.systemd.owner_uid"),
@@ -45,21 +45,21 @@ var journaldEventFields = FieldConversion{
 	"_UDEV_DEVLINK":                              text("journald.kernel.device_symlinks"),
 	"_UDEV_DEVNODE":                              text("journald.kernel.device_node_path"),
 	"_UDEV_SYSNAME":                              text("journald.kernel.device_name"),
-	sdjournal.SD_JOURNAL_FIELD_AUDIT_LOGINUID:    integer("process.audit.login_uid"),
-	sdjournal.SD_JOURNAL_FIELD_AUDIT_SESSION:     text("process.audit.session"),
-	sdjournal.SD_JOURNAL_FIELD_BOOT_ID:           text("host.boot_id"),
-	sdjournal.SD_JOURNAL_FIELD_CAP_EFFECTIVE:     text("process.capabilites"),
-	sdjournal.SD_JOURNAL_FIELD_CMDLINE:           text("process.cmd"),
+	sdjournal.SD_JOURNAL_FIELD_AUDIT_LOGINUID:    integer("journald.audit.login_uid"),
+	sdjournal.SD_JOURNAL_FIELD_AUDIT_SESSION:     text("journald.audit.session"),
+	sdjournal.SD_JOURNAL_FIELD_BOOT_ID:           text("journald.host.boot_id"),
+	sdjournal.SD_JOURNAL_FIELD_CAP_EFFECTIVE:     text("journald.process.capabilites"),
+	sdjournal.SD_JOURNAL_FIELD_CMDLINE:           text("journald.process.command_line"),
 	sdjournal.SD_JOURNAL_FIELD_CODE_FILE:         text("journald.code.file"),
 	sdjournal.SD_JOURNAL_FIELD_CODE_FUNC:         text("journald.code.func"),
 	sdjournal.SD_JOURNAL_FIELD_CODE_LINE:         integer("journald.code.line"),
-	sdjournal.SD_JOURNAL_FIELD_COMM:              text("process.name"),
-	sdjournal.SD_JOURNAL_FIELD_EXE:               text("process.executable"),
-	sdjournal.SD_JOURNAL_FIELD_GID:               integer("process.uid"),
+	sdjournal.SD_JOURNAL_FIELD_COMM:              text("journald.process.name"),
+	sdjournal.SD_JOURNAL_FIELD_EXE:               text("journald.process.executable"),
+	sdjournal.SD_JOURNAL_FIELD_GID:               integer("journald.gid"),
 	sdjournal.SD_JOURNAL_FIELD_HOSTNAME:          text("host.hostname"),
 	sdjournal.SD_JOURNAL_FIELD_MACHINE_ID:        text("host.id"),
 	sdjournal.SD_JOURNAL_FIELD_MESSAGE:           text("message"),
-	sdjournal.SD_JOURNAL_FIELD_PID:               integer("process.pid"),
+	sdjournal.SD_JOURNAL_FIELD_PID:               integer("journald.pid"),
 	sdjournal.SD_JOURNAL_FIELD_PRIORITY:          integer("syslog.priority", "log.syslog.priority"),
 	sdjournal.SD_JOURNAL_FIELD_SYSLOG_FACILITY:   integer("syslog.facility", "log.syslog.facility.name"),
 	sdjournal.SD_JOURNAL_FIELD_SYSLOG_IDENTIFIER: text("syslog.identifier"),
@@ -71,7 +71,7 @@ var journaldEventFields = FieldConversion{
 	sdjournal.SD_JOURNAL_FIELD_SYSTEMD_UNIT:      text("systemd.unit"),
 	sdjournal.SD_JOURNAL_FIELD_SYSTEMD_USER_UNIT: text("systemd.user_unit"),
 	sdjournal.SD_JOURNAL_FIELD_TRANSPORT:         text("systemd.transport"),
-	sdjournal.SD_JOURNAL_FIELD_UID:               integer("process.uid"),
+	sdjournal.SD_JOURNAL_FIELD_UID:               integer("journald.uid"),
 
 	// docker journald fields from: https://docs.docker.com/config/containers/logging/journald/
 	"CONTAINER_ID":              text("container.id_truncated"),

journalbeat/pkg/journalfield/default_other.go

@@ -26,9 +26,9 @@ var journaldEventFields = FieldConversion{
 	"COREDUMP_USER_UNIT":       text("journald.coredump.user_unit"),
 	"OBJECT_AUDIT_LOGINUID":    integer("journald.object.audit.login_uid"),
 	"OBJECT_AUDIT_SESSION":     integer("journald.object.audit.session"),
-	"OBJECT_CMDLINE":           text("journald.object.cmd"),
-	"OBJECT_COMM":              text("journald.object.name"),
-	"OBJECT_EXE":               text("journald.object.executable"),
+	"OBJECT_CMDLINE":           text("journald.object.process.command_line"),
+	"OBJECT_COMM":              text("journald.object.process.name"),
+	"OBJECT_EXE":               text("journald.object.process.executable"),
 	"OBJECT_GID":               integer("journald.object.gid"),
 	"OBJECT_PID":               integer("journald.object.pid"),
 	"OBJECT_SYSTEMD_OWNER_UID": integer("journald.object.systemd.owner_uid"),