From a90174c5224a638bcac123aec8839e257c7d44b8 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Fri, 22 Mar 2019 18:21:43 +0000 Subject: [PATCH 1/7] Truncate and base64 entity hash. --- x-pack/auditbeat/module/system/entity_hash.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/x-pack/auditbeat/module/system/entity_hash.go b/x-pack/auditbeat/module/system/entity_hash.go index ce968a7b461..1927d928eb2 100644 --- a/x-pack/auditbeat/module/system/entity_hash.go +++ b/x-pack/auditbeat/module/system/entity_hash.go @@ -6,7 +6,7 @@ package system import ( "crypto/sha256" - "encoding/hex" + "encoding/base64" "hash" ) @@ -20,7 +20,12 @@ func NewEntityHash() EntityHash { return EntityHash{sha256.New()} } -// Sum returns the hash as a string. +// Sum returns the base64 representation of the hash, +// truncated to 12 bytes. func (h *EntityHash) Sum() string { - return hex.EncodeToString(h.Hash.Sum(nil)) + hash := h.Hash.Sum(nil) + if len(hash) > 12 { + hash = hash[:12] + } + return base64.RawStdEncoding.EncodeToString(hash) } From 4598c40be40fb149758f5b59e2843b6caa12e5e2 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Fri, 22 Mar 2019 18:22:09 +0000 Subject: [PATCH 2/7] Package test data --- x-pack/auditbeat/module/system/package/_meta/data.json | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/x-pack/auditbeat/module/system/package/_meta/data.json b/x-pack/auditbeat/module/system/package/_meta/data.json index 5071146db3b..6d6b67342fb 100644 --- a/x-pack/auditbeat/module/system/package/_meta/data.json +++ b/x-pack/auditbeat/module/system/package/_meta/data.json @@ -1,13 +1,9 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { "action": "existing_package", "dataset": "package", - "id": "ed069c3f-1d30-4e17-845b-cc915cf108b4", + "id": "5cafbacd-9288-4022-be30-521db563f669", "kind": "state", "module": "system" }, @@ -18,11 +14,11 @@ "system": { "audit": { "package": { - "entity_id": "c2c455d9f99375d9fefc61da52fa93778976d54b1964164778058980531d77dc", + "entity_id": "wsRV2fmTddn+/GHa", "installtime": "2018-08-30T18:41:23.85657356+01:00", "name": "zstd", "summary": "Zstandard is a real-time compression algorithm", - "url": "http://zstd.net/", + "url": "https://facebook.github.io/zstd/", "version": "1.3.5" } } From b69cd0eb9cff18098c79c724e515561751873298 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Fri, 22 Mar 2019 18:25:13 +0000 Subject: [PATCH 3/7] Update process test data and test. --- .../module/system/process/_meta/data.json | 20 ++--- .../module/system/process/process_test.go | 80 ++++++++++--------- 2 files changed, 52 insertions(+), 48 deletions(-) diff --git a/x-pack/auditbeat/module/system/process/_meta/data.json b/x-pack/auditbeat/module/system/process/_meta/data.json index dc7d3b23d5a..abd1e7d5d68 100644 --- a/x-pack/auditbeat/module/system/process/_meta/data.json +++ b/x-pack/auditbeat/module/system/process/_meta/data.json @@ -1,27 +1,23 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { - "action": "process_started", + "action": "process_stopped", "dataset": "process", "kind": "event", "module": "system" }, - "message": "Process zsh (PID: 12936) by user elastic STARTED", + "message": "Process zsh (PID: 9086) by user elastic STOPPED", "process": { "args": [ "zsh" ], - "entity_id": "e2e0c5f51b093b71afed6af23debc906090d2f2f2afa9b930bf7e0803a6b53d5", + "entity_id": "+fYshazplsMYlr0y", "executable": "/bin/zsh", "name": "zsh", - "pid": 12936, - "ppid": 3858, - "start": "2019-01-21T15:01:54.782288Z", - "working_directory": "/Users/elastic" + "pid": 9086, + "ppid": 9085, + "start": "2019-01-01T00:00:01Z", + "working_directory": "/home/elastic" }, "service": { "type": "system" @@ -46,4 +42,4 @@ "id": "1000" } } -} +} \ No newline at end of file diff --git a/x-pack/auditbeat/module/system/process/process_test.go b/x-pack/auditbeat/module/system/process/process_test.go index d5aed7306fa..a0999947a11 100644 --- a/x-pack/auditbeat/module/system/process/process_test.go +++ b/x-pack/auditbeat/module/system/process/process_test.go @@ -22,6 +22,12 @@ func TestData(t *testing.T) { defer abtest.SetupDataDir(t)() f := mbtest.NewReportingMetricSetV2(t, getConfig()) + + // Set lastState and add test process to cache so it will be reported as stopped. + f.(*MetricSet).lastState = time.Now() + p := testProcess() + f.(*MetricSet).cache.DiffAndUpdateCache(convertToCacheable([]*Process{p})) + events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("received error: %+v", errs[0]) @@ -31,7 +37,7 @@ func TestData(t *testing.T) { t.Fatal("no events were generated") } - fullEvent := mbtest.StandardizeEvent(f, events[0], core.AddDatasetToEvent) + fullEvent := mbtest.StandardizeEvent(f, events[len(events)-1], core.AddDatasetToEvent) mbtest.WriteEventToDataJSON(t, fullEvent, "") } @@ -45,37 +51,9 @@ func getConfig() map[string]interface{} { func TestProcessEvent(t *testing.T) { ms := mbtest.NewReportingMetricSetV2(t, getConfig()).(*MetricSet) - process := Process{ - Info: types.ProcessInfo{ - Name: "zsh", - PID: 9086, - PPID: 9085, - CWD: "/home/elastic", - Exe: "/bin/zsh", - Args: []string{"zsh"}, - StartTime: time.Date(2019, 1, 1, 0, 0, 1, 0, time.UTC), - }, - UserInfo: &types.UserInfo{ - UID: "1002", - EUID: "1002", - SUID: "1002", - GID: "1002", - EGID: "1002", - SGID: "1002", - }, - User: &user.User{ - Uid: "1002", - Username: "elastic", - }, - Group: &user.Group{ - Gid: "1002", - Name: "elastic", - }, - } eventType := eventTypeEvent eventAction := eventActionProcessStarted - - event := ms.processEvent(&process, eventType, eventAction) + event := ms.processEvent(testProcess(), eventType, eventAction) containsError, err := event.RootFields.HasKey("error") if assert.NoError(t, err) { @@ -94,14 +72,14 @@ func TestProcessEvent(t *testing.T) { "process.args": []string{"zsh"}, "process.start": "2019-01-01 00:00:01 +0000 UTC", - "user.id": "1002", + "user.id": "1000", "user.name": "elastic", - "user.group.id": "1002", + "user.group.id": "1000", "user.group.name": "elastic", - "user.effective.id": "1002", - "user.effective.group.id": "1002", - "user.saved.id": "1002", - "user.saved.group.id": "1002", + "user.effective.id": "1000", + "user.effective.group.id": "1000", + "user.saved.id": "1000", + "user.saved.group.id": "1000", } for expFieldName, expFieldValue := range expectedRootFields { value, err := event.RootFields.GetValue(expFieldName) @@ -116,6 +94,36 @@ func TestProcessEvent(t *testing.T) { } } +func testProcess() *Process { + return &Process{ + Info: types.ProcessInfo{ + Name: "zsh", + PID: 9086, + PPID: 9085, + CWD: "/home/elastic", + Exe: "/bin/zsh", + Args: []string{"zsh"}, + StartTime: time.Date(2019, 1, 1, 0, 0, 1, 0, time.UTC), + }, + UserInfo: &types.UserInfo{ + UID: "1000", + EUID: "1000", + SUID: "1000", + GID: "1000", + EGID: "1000", + SGID: "1000", + }, + User: &user.User{ + Uid: "1000", + Username: "elastic", + }, + Group: &user.Group{ + Gid: "1000", + Name: "elastic", + }, + } +} + func TestPutIfNotEmpty(t *testing.T) { mapstr := common.MapStr{} From 439bd81c2d3472bf565dc9f5dd3c2f1fe3d3bc92 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Fri, 22 Mar 2019 18:37:13 +0000 Subject: [PATCH 4/7] Update socket test data and test. --- .../module/system/socket/_meta/data.json | 15 ++++-------- .../module/system/socket/socket_test.go | 24 +++++++++++++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/x-pack/auditbeat/module/system/socket/_meta/data.json b/x-pack/auditbeat/module/system/socket/_meta/data.json index 6ae45ea0863..60c87c66d3f 100644 --- a/x-pack/auditbeat/module/system/socket/_meta/data.json +++ b/x-pack/auditbeat/module/system/socket/_meta/data.json @@ -1,21 +1,16 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "destination": { "ip": "10.0.2.15", "port": 22 }, "event": { - "action": "existing_socket", + "action": "socket_closed", "dataset": "socket", - "id": "6aff69f8-7267-4604-9701-d7b67a7c65bc", - "kind": "state", + "kind": "event", "module": "system" }, - "message": "Inbound socket (10.0.2.2:55270 -\u003e 10.0.2.15:22) OPEN by process sshd (PID: 22799) and user root (UID: 0)", + "message": "Inbound socket (10.0.2.2:55270 -\u003e 10.0.2.15:22) CLOSED by process sshd (PID: 22799) and user root (UID: 0)", "network": { "direction": "inbound", "type": "ipv4" @@ -28,7 +23,7 @@ "type": "system" }, "socket": { - "entity_id": "d85bf25935c0ebbabc053024d4954ddd78979bd1390364ac46395c390ed7a6df" + "entity_id": "fIj66YRoGyoe8dML" }, "source": { "ip": "10.0.2.2", @@ -38,4 +33,4 @@ "id": 0, "name": "root" } -} +} \ No newline at end of file diff --git a/x-pack/auditbeat/module/system/socket/socket_test.go b/x-pack/auditbeat/module/system/socket/socket_test.go index 5588cccf16e..e7fbeeeb60b 100644 --- a/x-pack/auditbeat/module/system/socket/socket_test.go +++ b/x-pack/auditbeat/module/system/socket/socket_test.go @@ -12,19 +12,28 @@ import ( "strconv" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/elastic/beats/auditbeat/core" abtest "github.com/elastic/beats/auditbeat/testing" + sock "github.com/elastic/beats/metricbeat/helper/socket" "github.com/elastic/beats/metricbeat/mb" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/gosigar/sys/linux" ) func TestData(t *testing.T) { defer abtest.SetupDataDir(t)() f := mbtest.NewReportingMetricSetV2(t, getConfig()) + + // Set lastState and add test process to cache so it will be reported as stopped. + f.(*MetricSet).lastState = time.Now() + s := testSocket() + f.(*MetricSet).cache.DiffAndUpdateCache(convertToCacheable([]*Socket{s})) + events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("received error: %+v", errs[0]) @@ -39,6 +48,21 @@ func TestData(t *testing.T) { mbtest.WriteEventToDataJSON(t, fullEvent, "") } +func testSocket() *Socket { + return &Socket{ + Family: linux.AF_INET, + LocalIP: net.IPv4(10, 0, 2, 15), + LocalPort: 22, + RemoteIP: net.IPv4(10, 0, 2, 2), + RemotePort: 55270, + Direction: sock.Inbound, + UID: 0, + Username: "root", + ProcessPID: 22799, + ProcessName: "sshd", + } +} + func TestFetch(t *testing.T) { defer abtest.SetupDataDir(t)() From a505b41780bdc32e81f00a55e9c400f615d1d3ec Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Fri, 22 Mar 2019 19:30:00 +0000 Subject: [PATCH 5/7] Update user test data and test. --- .../module/system/user/_meta/data.json | 32 ++++++--------- x-pack/auditbeat/module/system/user/user.go | 10 +++-- .../auditbeat/module/system/user/user_test.go | 39 +++++++++++++++++-- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/x-pack/auditbeat/module/system/user/_meta/data.json b/x-pack/auditbeat/module/system/user/_meta/data.json index 65a49cd5073..963c64f3d89 100644 --- a/x-pack/auditbeat/module/system/user/_meta/data.json +++ b/x-pack/auditbeat/module/system/user/_meta/data.json @@ -1,17 +1,12 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { - "action": "existing_user", + "action": "user_added", "dataset": "user", - "id": "11b3b49c-79a1-4983-aea9-3257a3073a71", - "kind": "state", + "kind": "event", "module": "system" }, - "message": "Existing user elastic (UID: 1002, Groups: elastic,docker)", + "message": "New user elastic (UID: 1001, Groups: elastic,docker)", "service": { "type": "system" }, @@ -19,31 +14,26 @@ "audit": { "user": { "dir": "/home/elastic", - "gid": "1002", + "gid": "1001", "group": [ { - "gid": "1002", + "gid": "1001", "name": "elastic" }, { - "gid": "999", + "gid": "1002", "name": "docker" } ], "name": "elastic", - "password": { - "last_changed": "2018-12-07T00:00:00Z", - "type": "shadow_password" - }, - "shell": "/usr/bin/zsh", - "uid": "1002", - "user_information": ",,," + "shell": "/bin/bash", + "uid": "1001" } } }, "user": { - "entity_id": "4a80efe9ab38d1bb28aaa207d03ef24a702602d0ae55cc4661946fa1d8eee6b5", - "id": "1002", + "entity_id": "FgDfgeDptvvfdX+L", + "id": "1001", "name": "elastic" } -} +} \ No newline at end of file diff --git a/x-pack/auditbeat/module/system/user/user.go b/x-pack/auditbeat/module/system/user/user.go index 2086486a248..9e97717a198 100644 --- a/x-pack/auditbeat/module/system/user/user.go +++ b/x-pack/auditbeat/module/system/user/user.go @@ -466,10 +466,12 @@ func userMessage(user *User, action eventAction) string { func fmtGroups(groups []*user.Group) string { var b strings.Builder - b.WriteString(groups[0].Name) - for _, group := range groups[1:] { - b.WriteString(",") - b.WriteString(group.Name) + if len(groups) > 0 { + b.WriteString(groups[0].Name) + for _, group := range groups[1:] { + b.WriteString(",") + b.WriteString(group.Name) + } } return b.String() diff --git a/x-pack/auditbeat/module/system/user/user_test.go b/x-pack/auditbeat/module/system/user/user_test.go index 64b367322cf..4ad88015949 100644 --- a/x-pack/auditbeat/module/system/user/user_test.go +++ b/x-pack/auditbeat/module/system/user/user_test.go @@ -7,7 +7,9 @@ package user import ( + "os/user" "testing" + "time" "github.com/elastic/beats/auditbeat/core" abtest "github.com/elastic/beats/auditbeat/testing" @@ -18,6 +20,12 @@ func TestData(t *testing.T) { defer abtest.SetupDataDir(t)() f := mbtest.NewReportingMetricSetV2(t, getConfig()) + + // Set lastState and add test process to cache so it will be reported as stopped. + f.(*MetricSet).lastState = time.Now() + u := testUser() + f.(*MetricSet).cache.DiffAndUpdateCache(convertToCacheable([]*User{u})) + events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("received error: %+v", errs[0]) @@ -27,9 +35,34 @@ func TestData(t *testing.T) { t.Fatal("no events were generated") } - // The first user (events[0]) is usually root, the last one should be more interesting. - fullEvent := mbtest.StandardizeEvent(f, events[len(events)-1], core.AddDatasetToEvent) - mbtest.WriteEventToDataJSON(t, fullEvent, "") + for _, e := range events { + if name, _ := e.RootFields.GetValue("user.name"); name == "elastic" { + fullEvent := mbtest.StandardizeEvent(f, e, core.AddDatasetToEvent) + mbtest.WriteEventToDataJSON(t, fullEvent, "") + return + } + } + + t.Fatal("user not found") +} + +func testUser() *User { + return &User{ + Name: "elastic", + UID: "9999", + GID: "1001", + Groups: []*user.Group{ + &user.Group{ + Gid: "1001", + Name: "elastic", + }, + &user.Group{ + Gid: "1002", + Name: "docker", + }, + }, + Dir: "/home/elastic", + } } func getConfig() map[string]interface{} { From c212e1216bcf9b0a7b72d0afe64f1881d57be037 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Fri, 22 Mar 2019 21:07:38 +0000 Subject: [PATCH 6/7] Changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 16f43d97714..7de338d3173 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -50,6 +50,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename `event.type` to `auditd.message_type` in auditd module because event.type is reserved for future use by ECS. {pull}10536[10536] - Rename `auditd.messages` to `event.original` and `auditd.warnings` to `error.message`. {pull}10577[10577] - Process dataset: Only report processes with executable. {pull}11232[11232] +- Shorten entity IDs. *Filebeat* From 69e754aa2f1f79575a3848208c19fd0db6d9e14d Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Fri, 22 Mar 2019 21:31:10 +0000 Subject: [PATCH 7/7] Changelog --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7de338d3173..9adfde4a94a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -50,7 +50,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename `event.type` to `auditd.message_type` in auditd module because event.type is reserved for future use by ECS. {pull}10536[10536] - Rename `auditd.messages` to `event.original` and `auditd.warnings` to `error.message`. {pull}10577[10577] - Process dataset: Only report processes with executable. {pull}11232[11232] -- Shorten entity IDs. +- Shorten entity IDs. {pull}11405[11405] *Filebeat*