Skip to content

Commit

Permalink
Adjust ETW mapping to fit ECS
Browse files Browse the repository at this point in the history
  • Loading branch information
@chemamartinez
chemamartinez committed Feb 10, 2024
1 parent 198d928 commit 1ed5695
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 138 deletions.
36 changes: 9 additions & 27 deletions x-pack/filebeat/input/etw/_meta/fields.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
activity.
- name: channel
type: short
type: keyword
required: false
description: >
Used to enable special event processing. Channel values below 16 are reserved for use by Microsoft to enable special treatment by the ETW runtime. Channel values 16 and above will be ignored by the ETW runtime (treated the same as channel 0) and can be given user-defined semantics.
Expand All @@ -32,46 +32,34 @@
The event-specific data. The content of this object is specific to
any provider and event.
- name: event_id
type: short
required: true
description: >
The event identifier. The value is specific to the source of the event.
- name: flags
type: short
type: keyword
required: false
description: >
Flags that provide information about the event such as the type of session it was logged to and if the event contains extended data.
- name: keywords
type: long
type: keyword
required: false
description: >
The keywords are used to indicate an event's membership in a set of event categories.
- name: level
type: short
required: false
description: >
Level of severity. Level values 0 through 5 are defined by Microsoft. Level values 6 through 15 are reserved. Level values 16 through 255 can be defined by the event provider.
- name: logfile
type: keyword
required: false
description: >
The source file from which events are logged. Only available for non real-time sessions.
Level of severity. Level values 0 through 5 are defined by Microsoft. Level values 6 through 15 are reserved. Level values 16 through 255 can be defined by the event provider.
- name: opcode
type: short
type: keyword
required: false
description: >
The opcode defined in the event. Task and opcode are typically used to
identify the location in the application from where the event was
logged.
- name: process_id
type: integer
type: keyword
required: false
description: >
Identifies the process that generated the event.
Expand Down Expand Up @@ -103,27 +91,21 @@
Human-readable level of severity.
- name: task
type: short
type: keyword
required: false
description: >
The task defined in the event. Task and opcode are typically used to
identify the location in the application from where the event was
logged.
- name: thread_id
type: integer
type: keyword
required: false
description: >
Identifies the thread that generated the event.
- name: timestamp
type: date
required: false
description: >
Contains the time that the event occurred.
- name: version
type: short
type: keyword
required: false
description: >
Specify the version of a manifest-based event.
82 changes: 43 additions & 39 deletions x-pack/filebeat/input/etw/input.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,13 +154,7 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error {
return 1
}

evt := beat.Event{
Timestamp: time.Now(),
Fields: mapstr.M{
"winlog": buildEvent(data, record.EventHeader, e.etwSession, e.config),
},
}

evt := buildEvent(data, record.EventHeader, e.etwSession, e.config)
publisher.Publish(evt)

return 0
Expand Down Expand Up @@ -198,7 +192,38 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error {
}

// buildEvent builds the winlog object.
func buildEvent(data map[string]any, h etw.EventHeader, session *etw.Session, cfg config) map[string]any {
func buildEvent(data map[string]any, h etw.EventHeader, session *etw.Session, cfg config) beat.Event {

winlog := map[string]any{
"activity_guid": h.ActivityId.String(),
"channel": fmt.Sprintf("%d", h.EventDescriptor.Channel),
"event_data": data,
"flags": fmt.Sprintf("%d", h.Flags),
"keywords": fmt.Sprintf("%d", h.EventDescriptor.Keyword),
"opcode": fmt.Sprintf("%d", h.EventDescriptor.Opcode),
"process_id": fmt.Sprintf("%d", h.ProcessId),
"provider_guid": h.ProviderId.String(),
"session": session.Name,
"task": fmt.Sprintf("%d", h.EventDescriptor.Task),
"thread_id": fmt.Sprintf("%d", h.ThreadId),
"version": fmt.Sprintf("%d", h.EventDescriptor.Version),
}

// Include provider GUID if not available in event header
zeroGUID := "{00000000-0000-0000-0000-000000000000}"
if winlog["provider_guid"] == zeroGUID {
winlog["provider_guid"] = session.GUID.String()
}

// Define fields map with Windows data and ECS mapping
fields := mapstr.M{
"winlog": winlog,
"event.code": fmt.Sprintf("%d", h.EventDescriptor.Id),
"event.created": time.Now(),
"event.kind": "event",
"event.severity": h.EventDescriptor.Level,
}

// Mapping from Level to Severity
levelToSeverity := map[uint8]string{
1: "critical",
Expand All @@ -209,46 +234,25 @@ func buildEvent(data map[string]any, h etw.EventHeader, session *etw.Session, cf
}

// Get the severity level, with a default value if not found
severity, ok := levelToSeverity[h.EventDescriptor.Level]
if !ok {
severity = "unknown" // Default severity level
_, ok := levelToSeverity[h.EventDescriptor.Level]
if ok {
fields["log.level"] = levelToSeverity[h.EventDescriptor.Level]
}

winlog := map[string]any{
"activity_guid": h.ActivityId.String(),
"channel": h.EventDescriptor.Channel,
"event_data": data,
"event_id": h.EventDescriptor.Id,
"flags": h.Flags,
"keywords": h.EventDescriptor.Keyword,
"level": h.EventDescriptor.Level,
"opcode": h.EventDescriptor.Opcode,
"process_id": h.ProcessId,
"provider_guid": h.ProviderId.String(),
"session": session.Name,
"severity": severity,
"task": h.EventDescriptor.Task,
"thread_id": h.ThreadId,
"timestamp": convertFileTimeToGoTime(uint64(h.TimeStamp)),
"version": h.EventDescriptor.Version,
}

// Include provider name and GUID if available
// Include provider name if available
if cfg.ProviderName != "" {
winlog["provider_name"] = cfg.ProviderName
}

zeroGUID := "{00000000-0000-0000-0000-000000000000}"
if winlog["provider_guid"] == zeroGUID {
winlog["provider_guid"] = session.GUID.String()
fields["event.provider"] = cfg.ProviderName
}

// Include logfile path if available
if cfg.Logfile != "" {
winlog["logfile"] = cfg.Logfile
fields["log.file.path"] = cfg.Logfile
}

return winlog
return beat.Event{
Timestamp: convertFileTimeToGoTime(uint64(h.TimeStamp)),
Fields: fields,
}
}

// convertFileTimeToGoTime converts a Windows FileTime to a Go time.Time structure.
Expand Down
134 changes: 62 additions & 72 deletions x-pack/filebeat/input/etw/input_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
input "github.com/elastic/beats/v7/filebeat/input/v2"
"github.com/elastic/beats/v7/x-pack/libbeat/reader/etw"
"github.com/elastic/elastic-agent-libs/logp"
"github.com/elastic/elastic-agent-libs/mapstr"

"golang.org/x/sys/windows"
)
Expand Down Expand Up @@ -303,7 +304,7 @@ func Test_buildEvent(t *testing.T) {
header etw.EventHeader
session *etw.Session
cfg config
expected map[string]any
expected mapstr.M
}{
{
name: "TestStandardData",
Expand Down Expand Up @@ -354,27 +355,27 @@ func Test_buildEvent(t *testing.T) {
ProviderName: "TestProvider",
},

expected: map[string]any{
"activity_guid": "{12345678-1234-1234-1234-123456789ABC}",
"channel": uint8(10),
"event_data": map[string]any{
"key": "value",
expected: mapstr.M{
"winlog": map[string]any{
"activity_guid": "{12345678-1234-1234-1234-123456789ABC}",
"channel": "10",
"event_data": map[string]any{
"key": "value",
},
"flags": "30",
"keywords": "40",
"opcode": "50",
"process_id": "60",
"provider_guid": "{12345678-1234-1234-1234-123456789ABC}",
"session": "Elastic-TestProvider",
"task": "70",
"thread_id": "80",
"version": "90",
},
"event_id": uint16(20),
"flags": uint16(30),
"keywords": uint64(40),
"level": uint8(1),
"logfile": nil,
"opcode": uint8(50),
"process_id": uint32(60),
"provider_guid": "{12345678-1234-1234-1234-123456789ABC}",
"provider_name": "TestProvider",
"session": "Elastic-TestProvider",
"severity": "critical",
"task": uint16(70),
"thread_id": uint32(80),
"timestamp": "2024-02-05T22:03:09.035Z",
"version": uint8(90),
"event.code": "20",
"event.provider": "TestProvider",
"event.severity": uint8(1),
"log.level": "critical",
},
},
{
Expand Down Expand Up @@ -422,64 +423,53 @@ func Test_buildEvent(t *testing.T) {
Logfile: "C:\\TestFile",
},

expected: map[string]any{
"activity_guid": "{12345678-1234-1234-1234-123456789ABC}",
"channel": uint8(10),
"event_data": map[string]any{
"key": "value",
expected: mapstr.M{
"winlog": map[string]any{
"activity_guid": "{12345678-1234-1234-1234-123456789ABC}",
"channel": "10",
"event_data": map[string]any{
"key": "value",
},
"flags": "30",
"keywords": "40",
"opcode": "50",
"process_id": "60",
"provider_guid": "{12345678-1234-1234-1234-123456789ABC}",
"session": "Elastic-TestProvider",
"task": "70",
"thread_id": "80",
"version": "90",
},
"event_id": uint16(20),
"flags": uint16(30),
"keywords": uint64(40),
"level": uint8(17),
"logfile": "C:\\TestFile",
"opcode": uint8(50),
"process_id": uint32(60),
"provider_guid": "{12345678-1234-1234-1234-123456789ABC}",
"provider_name": "TestProvider",
"session": "Elastic-TestProvider",
"severity": "unknown",
"task": uint16(70),
"thread_id": uint32(80),
"timestamp": "2024-02-05T22:03:09.035Z",
"version": uint8(90),
"event.code": "20",
"event.provider": "TestProvider",
"event.severity": uint8(17),
"log.file.path": "C:\\TestFile",
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
winlog := buildEvent(tt.data, tt.header, tt.session, tt.cfg)

// Parse the expected time string to time.Time
expectedTime, err := time.Parse(time.RFC3339, tt.expected["timestamp"].(string))
if err != nil {
t.Fatalf("Failed to parse expected time string: %v", err)
}

winlogTime := winlog["timestamp"].(time.Time)

// Convert the expected time to the same time zone before comparison
expectedTime = expectedTime.In(winlogTime.Location())

assert.Equal(t, tt.expected["activity_guid"], winlog["activity_guid"])
assert.Equal(t, tt.expected["channel"], winlog["channel"])
assert.Equal(t, tt.expected["event_data"], winlog["event_data"])
assert.Equal(t, tt.expected["event_id"], winlog["event_id"])
assert.Equal(t, tt.expected["flags"], winlog["flags"])
assert.Equal(t, tt.expected["keywords"], winlog["keywords"])
assert.Equal(t, tt.expected["level"], winlog["level"])
assert.Equal(t, tt.expected["logfile"], winlog["logfile"])
assert.Equal(t, tt.expected["opcode"], winlog["opcode"])
assert.Equal(t, tt.expected["process_id"], winlog["process_id"])
assert.Equal(t, tt.expected["provider_guid"], winlog["provider_guid"])
assert.Equal(t, tt.expected["provider_name"], winlog["provider_name"])
assert.Equal(t, tt.expected["session"], winlog["session"])
assert.Equal(t, tt.expected["severity"], winlog["severity"])
assert.Equal(t, tt.expected["task"], winlog["task"])
assert.Equal(t, tt.expected["thread_id"], winlog["thread_id"])
assert.Equal(t, expectedTime, winlogTime)
assert.Equal(t, tt.expected["version"], winlog["version"])
evt := buildEvent(tt.data, tt.header, tt.session, tt.cfg)

assert.Equal(t, tt.expected["winlog"].(map[string]any)["activity_guid"], evt.Fields["winlog"].(map[string]any)["activity_guid"])
assert.Equal(t, tt.expected["winlog"].(map[string]any)["channel"], evt.Fields["winlog"].(map[string]any)["channel"])
assert.Equal(t, tt.expected["winlog"].(map[string]any)["event_data"], evt.Fields["winlog"].(map[string]any)["event_data"])
assert.Equal(t, tt.expected["winlog"].(map[string]any)["flags"], evt.Fields["winlog"].(map[string]any)["flags"])
assert.Equal(t, tt.expected["winlog"].(map[string]any)["keywords"], evt.Fields["winlog"].(map[string]any)["keywords"])
assert.Equal(t, tt.expected["winlog"].(map[string]any)["opcode"], evt.Fields["winlog"].(map[string]any)["opcode"])
assert.Equal(t, tt.expected["winlog"].(map[string]any)["process_id"], evt.Fields["winlog"].(map[string]any)["process_id"])
assert.Equal(t, tt.expected["winlog"].(map[string]any)["provider_guid"], evt.Fields["winlog"].(map[string]any)["provider_guid"])
assert.Equal(t, tt.expected["winlog"].(map[string]any)["session"], evt.Fields["winlog"].(map[string]any)["session"])
assert.Equal(t, tt.expected["winlog"].(map[string]any)["task"], evt.Fields["winlog"].(map[string]any)["task"])
assert.Equal(t, tt.expected["winlog"].(map[string]any)["thread_id"], evt.Fields["winlog"].(map[string]any)["thread_id"])
assert.Equal(t, tt.expected["winlog"].(map[string]any)["version"], evt.Fields["winlog"].(map[string]any)["version"])

assert.Equal(t, tt.expected["event.code"], evt.Fields["event.code"])
assert.Equal(t, tt.expected["event.provider"], evt.Fields["event.provider"])
assert.Equal(t, tt.expected["event.severity"], evt.Fields["event.severity"])
assert.Equal(t, tt.expected["log.file.path"], evt.Fields["log.file.path"])
assert.Equal(t, tt.expected["log.level"], evt.Fields["log.level"])

})
}
Expand Down

0 comments on commit 1ed5695

Please sign in to comment.