[Auditbeat] System module: Shorten entity ids (elastic#11405)

As discussed in elastic#11348, this shortens down entity IDs to base64-encoded 12 bytes. It makes quite the difference. I've also used the opportunity to finally standardize the `TestData` functions to (hopefully) always generate the same data (or at least to get closer to that), regardless of which system they are run on. Resolves elastic#11348. (cherry picked from commit 2f51a07)
cwurm · Mar 26, 2019 · 0f6c748 · 0f6c748
1 parent 95f6198
commit 0f6c748
Show file tree

Hide file tree

Showing 10 changed files with 146 additions and 96 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -21,6 +21,7 @@ https://github.com/elastic/beats/compare/v7.0.0-beta1...master[Check the HEAD di
 *Auditbeat*
 
 - Process dataset: Only report processes with executable. {pull}11232[11232]
+- Shorten entity IDs. {pull}11405[11405]
 
 *Filebeat*
 

diff --git a/x-pack/auditbeat/module/system/entity_hash.go b/x-pack/auditbeat/module/system/entity_hash.go
@@ -6,7 +6,7 @@ package system
 
 import (
 	"crypto/sha256"
-	"encoding/hex"
+	"encoding/base64"
 	"hash"
 )
 
@@ -20,7 +20,12 @@ func NewEntityHash() EntityHash {
 	return EntityHash{sha256.New()}
 }
 
-// Sum returns the hash as a string.
+// Sum returns the base64 representation of the hash,
+// truncated to 12 bytes.
 func (h *EntityHash) Sum() string {
-	return hex.EncodeToString(h.Hash.Sum(nil))
+	hash := h.Hash.Sum(nil)
+	if len(hash) > 12 {
+		hash = hash[:12]
+	}
+	return base64.RawStdEncoding.EncodeToString(hash)
 }
diff --git a/x-pack/auditbeat/module/system/package/_meta/data.json b/x-pack/auditbeat/module/system/package/_meta/data.json
@@ -1,13 +1,9 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "agent": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
-    },
     "event": {
         "action": "existing_package",
         "dataset": "package",
-        "id": "ed069c3f-1d30-4e17-845b-cc915cf108b4",
+        "id": "5cafbacd-9288-4022-be30-521db563f669",
         "kind": "state",
         "module": "system"
     },
@@ -18,11 +14,11 @@
     "system": {
         "audit": {
             "package": {
-                "entity_id": "c2c455d9f99375d9fefc61da52fa93778976d54b1964164778058980531d77dc",
+                "entity_id": "wsRV2fmTddn+/GHa",
                 "installtime": "2018-08-30T18:41:23.85657356+01:00",
                 "name": "zstd",
                 "summary": "Zstandard is a real-time compression algorithm",
-                "url": "http://zstd.net/",
+                "url": "https://facebook.github.io/zstd/",
                 "version": "1.3.5"
             }
         }

diff --git a/x-pack/auditbeat/module/system/process/_meta/data.json b/x-pack/auditbeat/module/system/process/_meta/data.json
@@ -1,27 +1,23 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "agent": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
-    },
     "event": {
-        "action": "process_started",
+        "action": "process_stopped",
         "dataset": "process",
         "kind": "event",
         "module": "system"
     },
-    "message": "Process zsh (PID: 12936) by user elastic STARTED",
+    "message": "Process zsh (PID: 9086) by user elastic STOPPED",
     "process": {
         "args": [
             "zsh"
         ],
-        "entity_id": "e2e0c5f51b093b71afed6af23debc906090d2f2f2afa9b930bf7e0803a6b53d5",
+        "entity_id": "+fYshazplsMYlr0y",
         "executable": "/bin/zsh",
         "name": "zsh",
-        "pid": 12936,
-        "ppid": 3858,
-        "start": "2019-01-21T15:01:54.782288Z",
-        "working_directory": "/Users/elastic"
+        "pid": 9086,
+        "ppid": 9085,
+        "start": "2019-01-01T00:00:01Z",
+        "working_directory": "/home/elastic"
     },
     "service": {
         "type": "system"
@@ -46,4 +42,4 @@
             "id": "1000"
         }
     }
-}
+}
diff --git a/x-pack/auditbeat/module/system/process/process_test.go b/x-pack/auditbeat/module/system/process/process_test.go
@@ -22,6 +22,12 @@ func TestData(t *testing.T) {
 	defer abtest.SetupDataDir(t)()
 
 	f := mbtest.NewReportingMetricSetV2(t, getConfig())
+
+	// Set lastState and add test process to cache so it will be reported as stopped.
+	f.(*MetricSet).lastState = time.Now()
+	p := testProcess()
+	f.(*MetricSet).cache.DiffAndUpdateCache(convertToCacheable([]*Process{p}))
+
 	events, errs := mbtest.ReportingFetchV2(f)
 	if len(errs) > 0 {
 		t.Fatalf("received error: %+v", errs[0])
@@ -31,7 +37,7 @@ func TestData(t *testing.T) {
 		t.Fatal("no events were generated")
 	}
 
-	fullEvent := mbtest.StandardizeEvent(f, events[0], core.AddDatasetToEvent)
+	fullEvent := mbtest.StandardizeEvent(f, events[len(events)-1], core.AddDatasetToEvent)
 	mbtest.WriteEventToDataJSON(t, fullEvent, "")
 }
 
@@ -45,37 +51,9 @@ func getConfig() map[string]interface{} {
 func TestProcessEvent(t *testing.T) {
 	ms := mbtest.NewReportingMetricSetV2(t, getConfig()).(*MetricSet)
 
-	process := Process{
-		Info: types.ProcessInfo{
-			Name:      "zsh",
-			PID:       9086,
-			PPID:      9085,
-			CWD:       "/home/elastic",
-			Exe:       "/bin/zsh",
-			Args:      []string{"zsh"},
-			StartTime: time.Date(2019, 1, 1, 0, 0, 1, 0, time.UTC),
-		},
-		UserInfo: &types.UserInfo{
-			UID:  "1002",
-			EUID: "1002",
-			SUID: "1002",
-			GID:  "1002",
-			EGID: "1002",
-			SGID: "1002",
-		},
-		User: &user.User{
-			Uid:      "1002",
-			Username: "elastic",
-		},
-		Group: &user.Group{
-			Gid:  "1002",
-			Name: "elastic",
-		},
-	}
 	eventType := eventTypeEvent
 	eventAction := eventActionProcessStarted
-
-	event := ms.processEvent(&process, eventType, eventAction)
+	event := ms.processEvent(testProcess(), eventType, eventAction)
 
 	containsError, err := event.RootFields.HasKey("error")
 	if assert.NoError(t, err) {
@@ -94,14 +72,14 @@ func TestProcessEvent(t *testing.T) {
 		"process.args":       []string{"zsh"},
 		"process.start":      "2019-01-01 00:00:01 +0000 UTC",
 
-		"user.id":                 "1002",
+		"user.id":                 "1000",
 		"user.name":               "elastic",
-		"user.group.id":           "1002",
+		"user.group.id":           "1000",
 		"user.group.name":         "elastic",
-		"user.effective.id":       "1002",
-		"user.effective.group.id": "1002",
-		"user.saved.id":           "1002",
-		"user.saved.group.id":     "1002",
+		"user.effective.id":       "1000",
+		"user.effective.group.id": "1000",
+		"user.saved.id":           "1000",
+		"user.saved.group.id":     "1000",
 	}
 	for expFieldName, expFieldValue := range expectedRootFields {
 		value, err := event.RootFields.GetValue(expFieldName)
@@ -116,6 +94,36 @@ func TestProcessEvent(t *testing.T) {
 	}
 }
 
+func testProcess() *Process {
+	return &Process{
+		Info: types.ProcessInfo{
+			Name:      "zsh",
+			PID:       9086,
+			PPID:      9085,
+			CWD:       "/home/elastic",
+			Exe:       "/bin/zsh",
+			Args:      []string{"zsh"},
+			StartTime: time.Date(2019, 1, 1, 0, 0, 1, 0, time.UTC),
+		},
+		UserInfo: &types.UserInfo{
+			UID:  "1000",
+			EUID: "1000",
+			SUID: "1000",
+			GID:  "1000",
+			EGID: "1000",
+			SGID: "1000",
+		},
+		User: &user.User{
+			Uid:      "1000",
+			Username: "elastic",
+		},
+		Group: &user.Group{
+			Gid:  "1000",
+			Name: "elastic",
+		},
+	}
+}
+
 func TestPutIfNotEmpty(t *testing.T) {
 	mapstr := common.MapStr{}
 

diff --git a/x-pack/auditbeat/module/system/socket/_meta/data.json b/x-pack/auditbeat/module/system/socket/_meta/data.json
@@ -1,21 +1,16 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "agent": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
-    },
     "destination": {
         "ip": "10.0.2.15",
         "port": 22
     },
     "event": {
-        "action": "existing_socket",
+        "action": "socket_closed",
         "dataset": "socket",
-        "id": "6aff69f8-7267-4604-9701-d7b67a7c65bc",
-        "kind": "state",
+        "kind": "event",
         "module": "system"
     },
-    "message": "Inbound socket (10.0.2.2:55270 -\u003e 10.0.2.15:22) OPEN by process sshd (PID: 22799) and user root (UID: 0)",
+    "message": "Inbound socket (10.0.2.2:55270 -\u003e 10.0.2.15:22) CLOSED by process sshd (PID: 22799) and user root (UID: 0)",
     "network": {
         "direction": "inbound",
         "type": "ipv4"
@@ -28,7 +23,7 @@
         "type": "system"
     },
     "socket": {
-        "entity_id": "d85bf25935c0ebbabc053024d4954ddd78979bd1390364ac46395c390ed7a6df"
+        "entity_id": "fIj66YRoGyoe8dML"
     },
     "source": {
         "ip": "10.0.2.2",
@@ -38,4 +33,4 @@
         "id": 0,
         "name": "root"
     }
-}
+}
diff --git a/x-pack/auditbeat/module/system/socket/socket_test.go b/x-pack/auditbeat/module/system/socket/socket_test.go
@@ -12,19 +12,28 @@ import (
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 
 	"github.com/elastic/beats/auditbeat/core"
 	abtest "github.com/elastic/beats/auditbeat/testing"
+	sock "github.com/elastic/beats/metricbeat/helper/socket"
 	"github.com/elastic/beats/metricbeat/mb"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/gosigar/sys/linux"
 )
 
 func TestData(t *testing.T) {
 	defer abtest.SetupDataDir(t)()
 
 	f := mbtest.NewReportingMetricSetV2(t, getConfig())
+
+	// Set lastState and add test process to cache so it will be reported as stopped.
+	f.(*MetricSet).lastState = time.Now()
+	s := testSocket()
+	f.(*MetricSet).cache.DiffAndUpdateCache(convertToCacheable([]*Socket{s}))
+
 	events, errs := mbtest.ReportingFetchV2(f)
 	if len(errs) > 0 {
 		t.Fatalf("received error: %+v", errs[0])
@@ -39,6 +48,21 @@ func TestData(t *testing.T) {
 	mbtest.WriteEventToDataJSON(t, fullEvent, "")
 }
 
+func testSocket() *Socket {
+	return &Socket{
+		Family:      linux.AF_INET,
+		LocalIP:     net.IPv4(10, 0, 2, 15),
+		LocalPort:   22,
+		RemoteIP:    net.IPv4(10, 0, 2, 2),
+		RemotePort:  55270,
+		Direction:   sock.Inbound,
+		UID:         0,
+		Username:    "root",
+		ProcessPID:  22799,
+		ProcessName: "sshd",
+	}
+}
+
 func TestFetch(t *testing.T) {
 	defer abtest.SetupDataDir(t)()
 

diff --git a/x-pack/auditbeat/module/system/user/_meta/data.json b/x-pack/auditbeat/module/system/user/_meta/data.json
@@ -1,49 +1,39 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "agent": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
-    },
     "event": {
-        "action": "existing_user",
+        "action": "user_added",
         "dataset": "user",
-        "id": "11b3b49c-79a1-4983-aea9-3257a3073a71",
-        "kind": "state",
+        "kind": "event",
         "module": "system"
     },
-    "message": "Existing user elastic (UID: 1002, Groups: elastic,docker)",
+    "message": "New user elastic (UID: 1001, Groups: elastic,docker)",
     "service": {
         "type": "system"
     },
     "system": {
         "audit": {
             "user": {
                 "dir": "/home/elastic",
-                "gid": "1002",
+                "gid": "1001",
                 "group": [
                     {
-                        "gid": "1002",
+                        "gid": "1001",
                         "name": "elastic"
                     },
                     {
-                        "gid": "999",
+                        "gid": "1002",
                         "name": "docker"
                     }
                 ],
                 "name": "elastic",
-                "password": {
-                    "last_changed": "2018-12-07T00:00:00Z",
-                    "type": "shadow_password"
-                },
-                "shell": "/usr/bin/zsh",
-                "uid": "1002",
-                "user_information": ",,,"
+                "shell": "/bin/bash",
+                "uid": "1001"
             }
         }
     },
     "user": {
-        "entity_id": "4a80efe9ab38d1bb28aaa207d03ef24a702602d0ae55cc4661946fa1d8eee6b5",
-        "id": "1002",
+        "entity_id": "FgDfgeDptvvfdX+L",
+        "id": "1001",
         "name": "elastic"
     }
-}
+}
diff --git a/x-pack/auditbeat/module/system/user/user.go b/x-pack/auditbeat/module/system/user/user.go
@@ -466,10 +466,12 @@ func userMessage(user *User, action eventAction) string {
 func fmtGroups(groups []*user.Group) string {
 	var b strings.Builder
 
-	b.WriteString(groups[0].Name)
-	for _, group := range groups[1:] {
-		b.WriteString(",")
-		b.WriteString(group.Name)
+	if len(groups) > 0 {
+		b.WriteString(groups[0].Name)
+		for _, group := range groups[1:] {
+			b.WriteString(",")
+			b.WriteString(group.Name)
+		}
 	}
 
 	return b.String()