Merge branch 'feature-ecs-1.8' into o365-ecs1.8

CHANGELOG.next.asciidoc

@@ -841,6 +841,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added RFC6587 framing option for tcp and unix inputs {issue}23663[23663] {pull}23724[23724]
 - Upgrade Cisco ASA/FTD/Umbrella to ECS 1.8.0. {pull}23819[23819]
 - Add new ECS user and categories features to google_workspace/gsuite {issue}23118[23118] {pull}23709[23709]
+- Move crowdstrike JS processor to ingest pipelines and upgrade to ECS 1.8.0 {issue}23118[23118] {pull}23875[23875]
+- Update Filebeat auditd dataset to ECS 1.8.0. {pull}23723[23723] {issue}23118[23118]
+- Updated microsoft defender_atp and m365_defender to ECS 1.8. {pull}23897[23897] {issue}23118[23118]
 - Updated o365 module to ECS 1.8. {issue}23118[23118] {pull}23896[23896]
 
 *Heartbeat*

NOTICE.txt

@@ -6547,11 +6547,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-concert@v0.0
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-libaudit/v2
-Version: v2.1.0
+Version: v2.2.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.1.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.2.0/LICENSE.txt:
 
 
                                  Apache License

auditbeat/cmd/root.go

@@ -35,7 +35,7 @@ const (
 	Name = "auditbeat"
 
 	// ecsVersion specifies the version of ECS that Auditbeat is implementing.
-	ecsVersion = "1.7.0"
+	ecsVersion = "1.8.0"
 )
 
 // RootCmd for running auditbeat.

auditbeat/module/auditd/audit_linux.go

@@ -20,7 +20,6 @@ package auditd
 import (
 	"fmt"
 	"os"
-	"os/user"
 	"runtime"
 	"strconv"
 	"strings"
@@ -462,7 +461,7 @@ func filterRecordType(typ auparse.AuditMessageType) bool {
 	case typ == auparse.AUDIT_REPLACE:
 		return true
 	// Messages from 1300-2999 are valid audit message types.
-	case typ < auparse.AUDIT_USER_AUTH || typ > auparse.AUDIT_LAST_USER_MSG2:
+	case (typ < auparse.AUDIT_USER_AUTH || typ > auparse.AUDIT_LAST_USER_MSG2) && typ != auparse.AUDIT_LOGIN:
 		return true
 	}
 
@@ -554,35 +553,67 @@ func buildMetricbeatEvent(msgs []*auparse.AuditMessage, config Config) mb.Event
 
 	normalizeEventFields(auditEvent, out.RootFields)
 
-	switch auditEvent.Category {
-	case aucoalesce.EventTypeUserLogin:
-		// Set ECS user fields from the attempted login account.
-		if usernameOrID := auditEvent.Summary.Actor.Secondary; usernameOrID != "" {
-			if usr, err := resolveUsernameOrID(usernameOrID); err == nil {
-				out.RootFields.Put("user.name", usr.Username)
-				out.RootFields.Put("user.id", usr.Uid)
-			} else {
-				// The login account doesn't exists. Treat it as a user name
-				out.RootFields.Put("user.name", usernameOrID)
-				out.RootFields.Delete("user.id")
+	// User set for related.user
+	var userSet common.StringSet
+	if config.ResolveIDs {
+		userSet = make(common.StringSet)
+	}
+
+	// Copy user.*/group.* fields from event
+	setECSEntity := func(key string, ent aucoalesce.ECSEntityData, root common.MapStr, set common.StringSet) {
+		if ent.ID == "" && ent.Name == "" {
+			return
+		}
+		if ent.ID == uidUnset {
+			ent.ID = ""
+		}
+		nameField := key + ".name"
+		idField := key + ".id"
+		if ent.ID != "" {
+			root.Put(idField, ent.ID)
+		} else {
+			root.Delete(idField)
+		}
+		if ent.Name != "" {
+			root.Put(nameField, ent.Name)
+			if set != nil {
+				set.Add(ent.Name)
 			}
+		} else {
+			root.Delete(nameField)
 		}
 	}
 
-	return out
-}
+	setECSEntity("user", auditEvent.ECS.User.ECSEntityData, out.RootFields, userSet)
+	setECSEntity("user.effective", auditEvent.ECS.User.Effective, out.RootFields, userSet)
+	setECSEntity("user.target", auditEvent.ECS.User.Target, out.RootFields, userSet)
+	setECSEntity("user.changes", auditEvent.ECS.User.Changes, out.RootFields, userSet)
+	setECSEntity("group", auditEvent.ECS.Group, out.RootFields, nil)
 
-func resolveUsernameOrID(userOrID string) (usr *user.User, err error) {
-	usr, err = user.Lookup(userOrID)
-	if err == nil {
-		// User found by name
-		return
+	if userSet != nil {
+		if userSet.Count() != 0 {
+			out.RootFields.Put("related.user", userSet.ToSlice())
+		}
 	}
-	if _, ok := err.(user.UnknownUserError); !ok {
-		// Lookup failed by a reason other than user not found
-		return
+	getStringField := func(key string, m common.MapStr) (str string) {
+		if asIf, _ := m.GetValue(key); asIf != nil {
+			str, _ = asIf.(string)
+		}
+		return str
 	}
-	return user.LookupId(userOrID)
+
+	// Remove redundant user.effective.* when it's the same as user.*
+	removeRedundantEntity := func(target, original string, m common.MapStr) bool {
+		for _, suffix := range []string{".id", ".name"} {
+			if value := getStringField(original+suffix, m); value != "" && getStringField(target+suffix, m) == value {
+				m.Delete(target)
+				return true
+			}
+		}
+		return false
+	}
+	removeRedundantEntity("user.effective", "user", out.RootFields)
+	return out
 }
 
 func normalizeEventFields(event *aucoalesce.Event, m common.MapStr) {

auditbeat/module/auditd/audit_linux_test.go

@@ -24,7 +24,6 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"os/user"
 	"sort"
 	"strings"
 	"testing"
@@ -141,20 +140,20 @@ func TestLoginType(t *testing.T) {
 
 	for idx, expected := range []common.MapStr{
 		{
-			"event.category": []string{"authentication"},
-			"event.type":     []string{"start", "authentication_failure"},
-			"event.outcome":  "failure",
-			"user.name":      "(invalid user)",
-			"user.id":        nil,
-			"session":        nil,
+			"event.category":      []string{"authentication"},
+			"event.type":          []string{"start", "authentication_failure"},
+			"event.outcome":       "failure",
+			"user.effective.name": "(invalid user)",
+			"user.id":             nil,
+			"session":             nil,
 		},
 		{
-			"event.category": []string{"authentication"},
-			"event.type":     []string{"start", "authentication_success"},
-			"event.outcome":  "success",
-			"user.name":      "adrian",
-			"user.audit.id":  nil,
-			"auditd.session": nil,
+			"event.category":      []string{"authentication"},
+			"event.type":          []string{"start", "authentication_success"},
+			"event.outcome":       "success",
+			"user.effective.name": "adrian",
+			"user.audit.id":       nil,
+			"auditd.session":      nil,
 		},
 		{
 			"event.category": []string{"authentication"},
@@ -355,36 +354,12 @@ func assertNoErrors(t *testing.T, events []mb.Event) {
 	for _, e := range events {
 		t.Log(e)
 
-		if e.Error != nil {
+		if !assert.Nil(t, e.Error) {
 			t.Errorf("received error: %+v", e.Error)
 		}
-	}
-}
-
-func BenchmarkResolveUsernameOrID(b *testing.B) {
-	for _, query := range []struct {
-		input string
-		name  string
-		id    string
-		err   bool
-	}{
-		{input: "0", name: "root", id: "0"},
-		{input: "root", name: "root", id: "0"},
-		{input: "vagrant", name: "vagrant", id: "1000"},
-		{input: "1000", name: "vagrant", id: "1000"},
-		{input: "nonexisting", err: true},
-		{input: "9987", err: true},
-	} {
-		b.Run(query.input, func(b *testing.B) {
-			var usr *user.User
-			var err error
-			for i := 0; i < b.N; i++ {
-				usr, err = resolveUsernameOrID(query.input)
-			}
-			if assert.Equal(b, query.err, err != nil, fmt.Sprintf("%v", err)) && !query.err {
-				assert.Equal(b, query.name, usr.Username)
-				assert.Equal(b, query.id, usr.Uid)
-			}
-		})
+		errorMsgKey, err := e.RootFields.GetValue("error.message")
+		if err == nil && !assert.Nil(t, errorMsgKey) {
+			t.Errorf("event has error messages: %v", errorMsgKey)
+		}
 	}
 }