From 020d42a6ce736af64dc285e21c97764d433ff22d Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 20 Mar 2020 10:13:00 -0400 Subject: [PATCH] Add experimental event log reader with increased performance (#16849) (#17022) This PR adds a new event log reader implementation that's behind a feature flag for now. It achieves higher event throughput than the current reader by not using XML and by heavily caching static metadata about events. To enable it add `api` to each event log reader. ``` winlogbeat.event_logs: - name: Security api: wineventlog-experimental ``` The existing reader requests each event as XML and then must unmarshal the XML document. EvtFormatMessage is used to get the XML document from Windows. Then the Go stdlib encoder/xml package is used to parse it. Both of these operations are relatively slow (see https://github.com/golang/go/issues/21823 about encoding/xml). This new reader utilizes the publisher metadata APIs to fetch and cache metadata about all event IDs associated with a provider. It does this the first time it encounters a provider ID while reading events. __Risk: Caching this info could lead to having stale information in memory if metadata changes via software update (see Edge Cases).__ It caches the names of the event data parameters and a templatized version of the message string. To get the data for an event this reader receives EVT_VARIANT structs containing the parameters rather than receiving and parsing XML. This is more efficient because there are fewer and smaller memory allocations and no XML encoding or decoding. To get the message for an event it utilizes the cached text/template it has for the event ID and passes it the list of parameter values. Edge Cases There is no provider metadata installed on the host. Could happen for forwarded events or reading from .evtx files. - Mitigate by falling back to getting parameter names by the event XML and rendering the message with EvtFormatMessage for each event. Software is updated and an event ID changes it's event data parameters. Saw this between Sysmon versions 9 and 10 with event ID 5. - Mitigate by fingerprinting the number of event data parameters and their types. - If the fingerprint changes, fetch the XML for the event and store the parameter names. Benchmark Comparison Comparing batch_size 500, that's a 1396% increase in events/sec, a -81% reduction in bytes allocated per event, and -86% decrease in the number of allocations. PS C:\Gopath\src\github.com\elastic\beats\winlogbeat\eventlog> go test -run TestBenchmarkRead -benchmem -benchtime 10s -benchtest -v . --- PASS: TestBenchmarkRead (231.68s) --- PASS: TestBenchmarkRead/api=wineventlog (53.57s) --- PASS: TestBenchmarkRead/api=wineventlog/batch_size=10 (12.19s) bench_test.go:128: 2067.28 events/sec 18283 B/event 182836 B/batch 251 allocs/event 2516 allocs/batch --- PASS: TestBenchmarkRead/api=wineventlog/batch_size=100 (16.73s) bench_test.go:128: 2144.50 events/sec 17959 B/event 1795989 B/batch 250 allocs/event 25020 allocs/batch --- PASS: TestBenchmarkRead/api=wineventlog/batch_size=500 (13.48s) bench_test.go:128: 1888.40 events/sec 17648 B/event 8824455 B/batch 250 allocs/event 125018 allocs/batch --- PASS: TestBenchmarkRead/api=wineventlog/batch_size=1000 (11.18s) bench_test.go:128: 2064.14 events/sec 17650 B/event 17650459 B/batch 250 allocs/event 250012 allocs/batch --- PASS: TestBenchmarkRead/api=wineventlog-experimental (98.28s) --- PASS: TestBenchmarkRead/api=wineventlog-experimental/batch_size=10 (18.72s) bench_test.go:128: 16813.52 events/sec 3974 B/event 39744 B/batch 34 allocs/event 344 allocs/batch --- PASS: TestBenchmarkRead/api=wineventlog-experimental/batch_size=100 (25.39s) bench_test.go:128: 28300.30 events/sec 3634 B/event 363498 B/batch 33 allocs/event 3324 allocs/batch --- PASS: TestBenchmarkRead/api=wineventlog-experimental/batch_size=500 (26.40s) bench_test.go:128: 28266.73 events/sec 3332 B/event 1666041 B/batch 33 allocs/event 16597 allocs/batch --- PASS: TestBenchmarkRead/api=wineventlog-experimental/batch_size=1000 (27.77s) bench_test.go:128: 28387.74 events/sec 3330 B/event 3330690 B/batch 33 allocs/event 33127 allocs/batch --- PASS: TestBenchmarkRead/api=eventlogging (13.29s) bench_test.go:128: 56243.80 events/sec 8043 B/event 6513053 B/batch 31 allocs/event 25151 allocs/batch PASS ok github.com/elastic/beats/v7/winlogbeat/eventlog 231.932s (cherry picked from commit d81ef7368b0ef321ff33b42be877c4aa1f1c09e0) --- CHANGELOG.next.asciidoc | 1 + Vagrantfile | 5 + winlogbeat/docs/winlogbeat-options.asciidoc | 26 + winlogbeat/eventlog/bench_test.go | 135 ++-- winlogbeat/eventlog/eventlogging.go | 3 + winlogbeat/eventlog/eventlogging_test.go | 327 +++------ .../eventlog/wineventlog_expirimental.go | 298 ++++++++ winlogbeat/eventlog/wineventlog_test.go | 217 ++++-- winlogbeat/sys/event.go | 21 +- winlogbeat/sys/event_test.go | 1 + winlogbeat/sys/wineventlog/bookmark.go | 81 +++ winlogbeat/sys/wineventlog/bookmark_test.go | 88 +++ winlogbeat/sys/wineventlog/bufferpool.go | 113 +++ winlogbeat/sys/wineventlog/doc.go | 14 +- winlogbeat/sys/wineventlog/format_message.go | 102 +++ .../sys/wineventlog/format_message_test.go | 153 ++++ winlogbeat/sys/wineventlog/iterator.go | 207 ++++++ winlogbeat/sys/wineventlog/iterator_test.go | 271 +++++++ winlogbeat/sys/wineventlog/metadata_store.go | 463 ++++++++++++ .../sys/wineventlog/metadata_store_test.go | 63 ++ .../sys/wineventlog/publisher_metadata.go | 663 ++++++++++++++++++ .../wineventlog/publisher_metadata_test.go | 230 ++++++ winlogbeat/sys/wineventlog/query.go | 6 +- winlogbeat/sys/wineventlog/renderer.go | 437 ++++++++++++ winlogbeat/sys/wineventlog/renderer_test.go | 291 ++++++++ winlogbeat/sys/wineventlog/stringinserts.go | 85 +++ .../wineventlog/stringinserts_test.go} | 42 +- winlogbeat/sys/wineventlog/syscall_windows.go | 336 ++++++++- winlogbeat/sys/wineventlog/template.go | 35 + winlogbeat/sys/wineventlog/template_test.go | 43 ++ .../application-windows-error-reporting.evtx | Bin 0 -> 69632 bytes winlogbeat/sys/wineventlog/util_test.go | 153 ++++ .../wineventlog/wineventlog_windows_test.go | 4 - winlogbeat/sys/wineventlog/winmeta.go | 58 ++ .../sys/wineventlog/zsyscall_windows.go | 115 ++- 35 files changed, 4639 insertions(+), 448 deletions(-) create mode 100644 winlogbeat/eventlog/wineventlog_expirimental.go create mode 100644 winlogbeat/sys/wineventlog/bookmark.go create mode 100644 winlogbeat/sys/wineventlog/bookmark_test.go create mode 100644 winlogbeat/sys/wineventlog/bufferpool.go create mode 100644 winlogbeat/sys/wineventlog/format_message.go create mode 100644 winlogbeat/sys/wineventlog/format_message_test.go create mode 100644 winlogbeat/sys/wineventlog/iterator.go create mode 100644 winlogbeat/sys/wineventlog/iterator_test.go create mode 100644 winlogbeat/sys/wineventlog/metadata_store.go create mode 100644 winlogbeat/sys/wineventlog/metadata_store_test.go create mode 100644 winlogbeat/sys/wineventlog/publisher_metadata.go create mode 100644 winlogbeat/sys/wineventlog/publisher_metadata_test.go create mode 100644 winlogbeat/sys/wineventlog/renderer.go create mode 100644 winlogbeat/sys/wineventlog/renderer_test.go create mode 100644 winlogbeat/sys/wineventlog/stringinserts.go rename winlogbeat/{eventlog/common_test.go => sys/wineventlog/stringinserts_test.go} (50%) create mode 100644 winlogbeat/sys/wineventlog/template.go create mode 100644 winlogbeat/sys/wineventlog/template_test.go create mode 100644 winlogbeat/sys/wineventlog/testdata/application-windows-error-reporting.evtx create mode 100644 winlogbeat/sys/wineventlog/util_test.go create mode 100644 winlogbeat/sys/wineventlog/winmeta.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5cf98b41f5b..cd01556a321 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -339,6 +339,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add more DNS error codes to the Sysmon module. {issue}15685[15685] - Add Audit and Log Management, Computer Object Management, and Distribution Group related events to the Security module. {pull}15217[15217] +- Add experimental event log reader implementation that should be faster in most cases. {issue}6585[6585] {pull}16849[16849] ==== Deprecated diff --git a/Vagrantfile b/Vagrantfile index b7cc6f03523..eba786adda2 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -164,6 +164,11 @@ Vagrant.configure(2) do |config| config.vm.define "win2019", primary: true do |c| c.vm.box = "StefanScherer/windows_2019" c.vm.provision "shell", inline: $winPsProvision, privileged: false + + c.vm.provider :virtualbox do |vbox| + vbox.memory = 4096 + vbox.cpus = 4 + end end # Solaris 11.2 diff --git a/winlogbeat/docs/winlogbeat-options.asciidoc b/winlogbeat/docs/winlogbeat-options.asciidoc index 88cb61a553f..f5a7520e824 100644 --- a/winlogbeat/docs/winlogbeat-options.asciidoc +++ b/winlogbeat/docs/winlogbeat-options.asciidoc @@ -410,3 +410,29 @@ stopped. *{vista_and_newer}* Setting `no_more_events` to `stop` is useful when reading from archived event log files where you want to read the whole file then exit. There's a complete example of how to read from an `.evtx` file in the <>. + +[float] +==== `event_logs.api` + +experimental[] + +This selects the event log reader implementation that is used to read events +from the Windows APIs. You should only set this option when testing experimental +features. When the value is set to `wineventlog-experimental` Winlogbeat will +replace the default event log reader with the experimental implementation. +We are evaluating this implementation to see if it can provide increased +performance and reduce CPU usage. *{vista_and_newer}* + +[source,yaml] +-------------------------------------------------------------------------------- +winlogbeat.event_logs: + - name: ForwardedEvents + api: wineventlog-experimental +-------------------------------------------------------------------------------- + +There are a few notable differences in the events: + +* Events that contained data under `winlog.user_data` will now have it under + `winlog.event_data`. +* Setting `include_xml: true` has no effect. + diff --git a/winlogbeat/eventlog/bench_test.go b/winlogbeat/eventlog/bench_test.go index e665f01bbbd..ec680c5fb5d 100644 --- a/winlogbeat/eventlog/bench_test.go +++ b/winlogbeat/eventlog/bench_test.go @@ -22,95 +22,112 @@ package eventlog import ( "bytes" "flag" + "fmt" "math/rand" - "os/exec" "strconv" + "strings" "testing" - "time" - elog "github.com/andrewkroh/sys/windows/svc/eventlog" - "github.com/dustin/go-humanize" + "golang.org/x/sys/windows/svc/eventlog" + + "github.com/elastic/beats/v7/libbeat/common" ) -// Benchmark tests with customized output. (`go test -v -benchtime 10s -benchtest .`) +const gigabyte = 1 << 30 var ( - benchTest = flag.Bool("benchtest", false, "Run benchmarks for the eventlog package") - injectAmount = flag.Int("inject", 50000, "Number of events to inject before running benchmarks") + benchTest = flag.Bool("benchtest", false, "Run benchmarks for the eventlog package.") + injectAmount = flag.Int("inject", 1E6, "Number of events to inject before running benchmarks.") ) -// TestBenchmarkBatchReadSize tests the performance of different -// batch_read_size values. -func TestBenchmarkBatchReadSize(t *testing.T) { +// TestBenchmarkRead benchmarks each event log reader implementation with +// different batch sizes. +// +// Recommended usage: +// go test -run TestBenchmarkRead -benchmem -benchtime 10s -benchtest -v . +func TestBenchmarkRead(t *testing.T) { if !*benchTest { t.Skip("-benchtest not enabled") } - log, err := initLog(providerName, sourceName, eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t) + defer teardown() - // Increase the log size so that it can hold these large events. - output, err := exec.Command("wevtutil.exe", "sl", "/ms:1073741824", providerName).CombinedOutput() - if err != nil { - t.Fatal(err, string(output)) - } + setLogSize(t, providerName, gigabyte) // Publish test messages: for i := 0; i < *injectAmount; i++ { - err = log.Report(elog.Info, uint32(rand.Int63()%1000), []string{strconv.Itoa(i) + " " + randomSentence(256)}) + err := writer.Report(eventlog.Info, uint32(rand.Int63()%1000), []string{strconv.Itoa(i) + " " + randomSentence(256)}) if err != nil { - t.Fatal("ReportEvent error", err) + t.Fatal(err) } } - benchTest := func(batchSize int) { - var err error - result := testing.Benchmark(func(b *testing.B) { - eventlog, tearDown := setupWinEventLog(t, 0, map[string]interface{}{ - "name": providerName, - "batch_read_size": batchSize, - }) - defer tearDown() - b.ResetTimer() - - // Each iteration reads one batch. - for i := 0; i < b.N; i++ { - _, err = eventlog.Read() - if err != nil { - return - } + for _, api := range []string{winEventLogAPIName, winEventLogExpAPIName} { + t.Run("api="+api, func(t *testing.T) { + for _, batchSize := range []int{10, 100, 500, 1000} { + t.Run(fmt.Sprintf("batch_size=%d", batchSize), func(t *testing.T) { + result := testing.Benchmark(benchmarkEventLog(api, batchSize)) + outputBenchmarkResults(t, result) + }) } }) + } - if err != nil { - t.Fatal(err) - return + t.Run("api="+eventLoggingAPIName, func(t *testing.T) { + result := testing.Benchmark(benchmarkEventLog(eventLoggingAPIName, -1)) + outputBenchmarkResults(t, result) + }) +} + +func benchmarkEventLog(api string, batchSize int) func(b *testing.B) { + return func(b *testing.B) { + conf := common.MapStr{ + "name": providerName, + } + if strings.HasPrefix(api, "wineventlog") { + conf.Put("batch_read_size", batchSize) + conf.Put("no_more_events", "stop") } - t.Logf("batch_size=%v, total_events=%v, batch_time=%v, events_per_sec=%v, bytes_alloced_per_event=%v, total_allocs=%v", - batchSize, - result.N*batchSize, - time.Duration(result.NsPerOp()), - float64(batchSize)/time.Duration(result.NsPerOp()).Seconds(), - humanize.Bytes(result.MemBytes/(uint64(result.N)*uint64(batchSize))), - result.MemAllocs) - } + log := openLog(b, api, nil, conf) + defer log.Close() + + events := 0 + b.ResetTimer() + + // Each iteration reads one batch. + for i := 0; i < b.N; i++ { + records, err := log.Read() + if err != nil { + b.Fatal(err) + return + } + events += len(records) + } + + b.StopTimer() - benchTest(10) - benchTest(100) - benchTest(500) - benchTest(1000) + b.ReportMetric(float64(events), "events") + b.ReportMetric(float64(batchSize), "batch_size") + } } -// Utility Functions +func outputBenchmarkResults(t testing.TB, result testing.BenchmarkResult) { + totalBatches := result.N + totalEvents := int(result.Extra["events"]) + totalBytes := result.MemBytes + totalAllocs := result.MemAllocs + + eventsPerSec := float64(totalEvents) / result.T.Seconds() + bytesPerEvent := float64(totalBytes) / float64(totalEvents) + bytesPerBatch := float64(totalBytes) / float64(totalBatches) + allocsPerEvent := float64(totalAllocs) / float64(totalEvents) + allocsPerBatch := float64(totalAllocs) / float64(totalBatches) + + t.Logf("%.2f events/sec\t %d B/event\t %d B/batch\t %d allocs/event\t %d allocs/batch", + eventsPerSec, int(bytesPerEvent), int(bytesPerBatch), int(allocsPerEvent), int(allocsPerBatch)) +} var randomWords = []string{ "recover", diff --git a/winlogbeat/eventlog/eventlogging.go b/winlogbeat/eventlog/eventlogging.go index 3e9494c91b6..963797264b2 100644 --- a/winlogbeat/eventlog/eventlogging.go +++ b/winlogbeat/eventlog/eventlogging.go @@ -27,6 +27,7 @@ import ( "github.com/joeshaw/multierror" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/winlogbeat/checkpoint" "github.com/elastic/beats/v7/winlogbeat/sys" @@ -277,6 +278,8 @@ func (l *eventLogging) ignoreOlder(r *Record) bool { // newEventLogging creates and returns a new EventLog for reading event logs // using the Event Logging API. func newEventLogging(options *common.Config) (EventLog, error) { + cfgwarn.Deprecate("8.0", "The eventlogging API reader is deprecated.") + c := eventLoggingConfig{ ReadBufferSize: win.MaxEventBufferSize, FormatBufferSize: win.MaxFormatMessageBufferSize, diff --git a/winlogbeat/eventlog/eventlogging_test.go b/winlogbeat/eventlog/eventlogging_test.go index 7ed4ea78772..ba8524cad09 100644 --- a/winlogbeat/eventlog/eventlogging_test.go +++ b/winlogbeat/eventlog/eventlogging_test.go @@ -21,14 +21,11 @@ package eventlog import ( "fmt" - "os/exec" - "strconv" "strings" "sync" "testing" - elog "github.com/andrewkroh/sys/windows/svc/eventlog" - "github.com/joeshaw/multierror" + "github.com/andrewkroh/sys/windows/svc/eventlog" "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/logp" @@ -54,37 +51,33 @@ const ( netEventMsgFile = "%SystemRoot%\\System32\\netevent.dll" ) -const allLevels = elog.Success | elog.AuditFailure | elog.AuditSuccess | elog.Error | elog.Info | elog.Warning - -const gigabyte = 1 << 30 - // Test messages. var messages = map[uint32]struct { eventType uint16 message string }{ 1: { - eventType: elog.Info, + eventType: eventlog.Info, message: "Hmmmm.", }, 2: { - eventType: elog.Success, + eventType: eventlog.Success, message: "I am so blue I'm greener than purple.", }, 3: { - eventType: elog.Warning, + eventType: eventlog.Warning, message: "I stepped on a Corn Flake, now I'm a Cereal Killer.", }, 4: { - eventType: elog.Error, + eventType: eventlog.Error, message: "The quick brown fox jumps over the lazy dog.", }, 5: { - eventType: elog.AuditSuccess, + eventType: eventlog.AuditSuccess, message: "Where do random thoughts come from?", }, 6: { - eventType: elog.AuditFailure, + eventType: eventlog.AuditFailure, message: "Login failure for user xyz!", }, } @@ -100,107 +93,27 @@ func configureLogp() { } else { logp.DevelopmentSetup(logp.WithLevel(logp.WarnLevel)) } - - // Clear the event log before starting. - log, _ := elog.Open(sourceName) - eventlogging.ClearEventLog(eventlogging.Handle(log.Handle), "") - log.Close() }) } -// initLog initializes an event logger. It registers the source name with -// the registry if it does not already exist. -func initLog(provider, source, msgFile string) (*elog.Log, error) { - // Install entry to registry: - _, err := elog.Install(providerName, sourceName, msgFile, true, allLevels) - if err != nil { - return nil, err - } - - // Open a new logger for writing events: - log, err := elog.Open(sourceName) - if err != nil { - var errs multierror.Errors - errs = append(errs, err) - err := elog.RemoveSource(providerName, sourceName) - if err != nil { - errs = append(errs, err) - } - err = elog.RemoveProvider(providerName) - if err != nil { - errs = append(errs, err) - } - return nil, errs.Err() - } - - return log, nil -} - -// uninstallLog unregisters the event logger from the registry and closes the -// log's handle if it is open. -func uninstallLog(provider, source string, log *elog.Log) error { - var errs multierror.Errors - - if log != nil { - err := eventlogging.ClearEventLog(eventlogging.Handle(log.Handle), "") - if err != nil { - errs = append(errs, err) - } - - err = log.Close() - if err != nil { - errs = append(errs, err) - } - } - - err := elog.RemoveSource(providerName, sourceName) - if err != nil { - errs = append(errs, err) - } - - err = elog.RemoveProvider(providerName) - if err != nil { - errs = append(errs, err) - } - - return errs.Err() -} - -// setLogSize set the maximum number of bytes that an event log can hold. -func setLogSize(t testing.TB, provider string, sizeBytes int) { - output, err := exec.Command("wevtutil.exe", "sl", "/ms:"+strconv.Itoa(sizeBytes), providerName).CombinedOutput() - if err != nil { - t.Fatal("failed to set log size", err, string(output)) - } -} - // Verify that all messages are read from the event log. func TestRead(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t) + defer teardown() // Publish test messages: for k, m := range messages { - err = log.Report(m.eventType, k, []string{m.message}) - if err != nil { + if err := writer.Report(m.eventType, k, []string{m.message}); err != nil { t.Fatal(err) } } // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) - defer teardown() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -219,7 +132,7 @@ func TestRead(t *testing.T) { } // Validate getNumberOfEventLogRecords returns the correct number of messages. - numMessages, err := eventlogging.GetNumberOfEventLogRecords(eventlogging.Handle(log.Handle)) + numMessages, err := eventlogging.GetNumberOfEventLogRecords(eventlogging.Handle(writer.Handle)) assert.NoError(t, err) assert.Equal(t, len(messages), int(numMessages)) } @@ -229,36 +142,27 @@ func TestRead(t *testing.T) { // possible buffer so this error should not occur. func TestFormatMessageWithLargeMessage(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t) + defer teardown() - message := "Hello" - err = log.Report(elog.Info, 1, []string{message}) - if err != nil { + const message = "Hello" + if err := writer.Report(eventlog.Info, 1, []string{message}); err != nil { t.Fatal(err) } // Messages are received as UTF-16 so we must have enough space in the read // buffer for the message, a windows newline, and a null-terminator. - requiredBufferSize := len(message+"\r\n")*2 + 2 + const requiredBufferSize = len(message+"\r\n")*2 + 2 // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{ + log := openEventLogging(t, 0, map[string]interface{}{ "name": providerName, // Use a buffer smaller than what is required. "format_buffer_size": requiredBufferSize / 2, }) - defer teardown() + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -275,29 +179,20 @@ func TestFormatMessageWithLargeMessage(t *testing.T) { // insert strings (the message parameters) is returned. func TestReadUnknownEventId(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, servicesMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t, servicesMsgFile) + defer teardown() - var eventID uint32 = 1000 - msg := "Test Message" - err = log.Success(eventID, msg) - if err != nil { + const eventID uint32 = 1000 + const msg = "Test Message" + if err := writer.Success(eventID, msg); err != nil { t.Fatal(err) } // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) - defer teardown() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -319,30 +214,20 @@ func TestReadUnknownEventId(t *testing.T) { // of the files then the next file should be checked. func TestReadTriesMultipleEventMsgFiles(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, - servicesMsgFile+";"+eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t, servicesMsgFile, eventCreateMsgFile) + defer teardown() - var eventID uint32 = 1000 - msg := "Test Message" - err = log.Success(eventID, msg) - if err != nil { + const eventID uint32 = 1000 + const msg = "Test Message" + if err := writer.Success(eventID, msg); err != nil { t.Fatal(err) } // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) - defer teardown() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -359,34 +244,25 @@ func TestReadTriesMultipleEventMsgFiles(t *testing.T) { // Test event messages that require more than one message parameter. func TestReadMultiParameterMsg(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, servicesMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t, servicesMsgFile) + defer teardown() // EventID observed by exporting system event log to XML and doing calculation. // 7036 // 1073748860 = 16384 << 16 + 7036 // https://msdn.microsoft.com/en-us/library/windows/desktop/aa385206(v=vs.85).aspx - var eventID uint32 = 1073748860 - template := "The %s service entered the %s state." + const eventID uint32 = 1073748860 + const template = "The %s service entered the %s state." msgs := []string{"Windows Update", "running"} - err = log.Report(elog.Info, eventID, msgs) - if err != nil { + if err := writer.Report(eventlog.Info, eventID, msgs); err != nil { t.Fatal(err) } // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) - defer teardown() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -406,40 +282,31 @@ func TestReadMultiParameterMsg(t *testing.T) { func TestOpenInvalidProvider(t *testing.T) { configureLogp() - el := newTestEventLogging(t, map[string]interface{}{"name": "nonExistentProvider"}) - assert.NoError(t, el.Open(checkpoint.EventLogState{}), "Calling Open() on an unknown provider "+ - "should automatically open Application.") - _, err := el.Read() + log := openEventLogging(t, 0, map[string]interface{}{"name": "nonExistentProvider"}) + defer log.Close() + + _, err := log.Read() assert.NoError(t, err) } // Test event messages that require no parameters. func TestReadNoParameterMsg(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, netEventMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t, netEventMsgFile) + defer teardown() - var eventID uint32 = 2147489654 // 1<<31 + 6006 - template := "The Event log service was stopped." + const eventID uint32 = 2147489654 // 1<<31 + 6006 + const template = "The Event log service was stopped." msgs := []string{} - err = log.Report(elog.Info, eventID, msgs) - if err != nil { + if err := writer.Report(eventlog.Info, eventID, msgs); err != nil { t.Fatal(err) } // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) - defer teardown() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -458,33 +325,25 @@ func TestReadNoParameterMsg(t *testing.T) { // being cleared or reset while reading. func TestReadWhileCleared(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() - - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) + writer, teardown := createLog(t) defer teardown() - log.Info(1, "Message 1") - log.Info(2, "Message 2") - lr, err := eventlog.Read() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() + + writer.Info(1, "Message 1") + writer.Info(2, "Message 2") + lr, err := log.Read() assert.NoError(t, err, "Expected 2 messages but received error") assert.Len(t, lr, 2, "Expected 2 messages") - assert.NoError(t, eventlogging.ClearEventLog(eventlogging.Handle(log.Handle), "")) - lr, err = eventlog.Read() + assert.NoError(t, eventlogging.ClearEventLog(eventlogging.Handle(writer.Handle), "")) + lr, err = log.Read() assert.NoError(t, err, "Expected 0 messages but received error") assert.Len(t, lr, 0, "Expected 0 message") - log.Info(3, "Message 3") - lr, err = eventlog.Read() + writer.Info(3, "Message 3") + lr, err = log.Read() assert.NoError(t, err, "Expected 1 message but received error") assert.Len(t, lr, 1, "Expected 1 message") if len(lr) > 0 { @@ -493,34 +352,25 @@ func TestReadWhileCleared(t *testing.T) { } // Test event messages that include less parameters than required for message -// formating (caused a crash in previous versions) +// formatting (caused a crash in previous versions) func TestReadMissingParameters(t *testing.T) { configureLogp() - log, err := initLog(providerName, sourceName, servicesMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() + writer, teardown := createLog(t, servicesMsgFile) + defer teardown() - var eventID uint32 = 1073748860 + const eventID uint32 = 1073748860 // Missing parameters will be substituted by "(null)" - template := "The %s service entered the (null) state." + const template = "The %s service entered the (null) state." msgs := []string{"Windows Update"} - err = log.Report(elog.Info, eventID, msgs) - if err != nil { + if err := writer.Report(eventlog.Info, eventID, msgs); err != nil { t.Fatal(err) } // Read messages: - eventlog, teardown := setupEventLogging(t, 0, map[string]interface{}{"name": providerName}) - defer teardown() + log := openEventLogging(t, 0, map[string]interface{}{"name": providerName}) + defer log.Close() - records, err := eventlog.Read() + records, err := log.Read() if err != nil { t.Fatal(err) } @@ -535,22 +385,7 @@ func TestReadMissingParameters(t *testing.T) { strings.TrimRight(records[0].Message, "\r\n")) } -func newTestEventLogging(t *testing.T, options map[string]interface{}) EventLog { - return newTestEventLog(t, newEventLogging, options) +func openEventLogging(t *testing.T, recordID uint64, options map[string]interface{}) EventLog { + t.Helper() + return openLog(t, eventLoggingAPIName, &checkpoint.EventLogState{RecordNumber: recordID}, options) } - -func setupEventLogging(t *testing.T, recordID uint64, options map[string]interface{}) (EventLog, func()) { - return setupEventLog(t, newEventLogging, recordID, options) -} - -// TODO: Add more test cases: -// - Record number rollover (there may be an issue with this if ++ is used anywhere) -// - Reading from a source name instead of provider name (can't be done according to docs). -// - Persistent read mode shall support specifying a record number (or not specifying a record number). -// -- Invalid record number based on range (should start at first record). -// -- Invalid record number based on range timestamp match check (should start at first record). -// -- Valid record number -// --- Do not replay first record (it was already reported) -// -- First read (no saved state) should return the first record (send first reported record). -// - NewOnly read mode shall seek to end and ignore first. -// - ReadThenExit read mode shall seek to end, read backwards, honor the EOF, then exit. diff --git a/winlogbeat/eventlog/wineventlog_expirimental.go b/winlogbeat/eventlog/wineventlog_expirimental.go new file mode 100644 index 00000000000..5952aad0f5a --- /dev/null +++ b/winlogbeat/eventlog/wineventlog_expirimental.go @@ -0,0 +1,298 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package eventlog + +import ( + "io" + "os" + "path/filepath" + + "github.com/pkg/errors" + "go.uber.org/multierr" + "golang.org/x/sys/windows" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/winlogbeat/checkpoint" + win "github.com/elastic/beats/v7/winlogbeat/sys/wineventlog" +) + +const ( + // winEventLogExpApiName is the name used to identify the Windows Event Log API + // as both an event type and an API. + winEventLogExpAPIName = "wineventlog-experimental" +) + +// winEventLogExp implements the EventLog interface for reading from the Windows +// Event Log API. +type winEventLogExp struct { + config winEventLogConfig + query string + channelName string // Name of the channel from which to read. + file bool // Reading from file rather than channel. + maxRead int // Maximum number returned in one Read. + lastRead checkpoint.EventLogState // Record number of the last read event. + log *logp.Logger + + iterator *win.EventIterator + renderer *win.Renderer +} + +// Name returns the name of the event log (i.e. Application, Security, etc.). +func (l *winEventLogExp) Name() string { + return l.channelName +} + +func (l *winEventLogExp) Open(state checkpoint.EventLogState) error { + l.lastRead = state + + var err error + l.iterator, err = win.NewEventIterator( + win.WithSubscriptionFactory(func() (handle win.EvtHandle, err error) { + return l.open(l.lastRead) + }), + win.WithBatchSize(l.maxRead)) + return err +} + +func (l *winEventLogExp) open(state checkpoint.EventLogState) (win.EvtHandle, error) { + var bookmark win.Bookmark + if len(state.Bookmark) > 0 { + var err error + bookmark, err = win.NewBookmarkFromXML(state.Bookmark) + if err != nil { + return win.NilHandle, err + } + defer bookmark.Close() + } + + if l.file { + return l.openFile(state, bookmark) + } + return l.openChannel(bookmark) +} + +func (l *winEventLogExp) openChannel(bookmark win.Bookmark) (win.EvtHandle, error) { + // Using a pull subscription to receive events. See: + // https://msdn.microsoft.com/en-us/library/windows/desktop/aa385771(v=vs.85).aspx#pull + signalEvent, err := windows.CreateEvent(nil, 0, 0, nil) + if err != nil { + return win.NilHandle, err + } + defer windows.CloseHandle(signalEvent) + + var flags win.EvtSubscribeFlag + if bookmark > 0 { + flags = win.EvtSubscribeStartAfterBookmark + } else { + flags = win.EvtSubscribeStartAtOldestRecord + } + + l.log.Debugw("Using subscription query.", "winlog.query", l.query) + return win.Subscribe( + 0, // Session - nil for localhost + signalEvent, + "", // Channel - empty b/c channel is in the query + l.query, // Query - nil means all events + win.EvtHandle(bookmark), // Bookmark - for resuming from a specific event + flags) +} + +func (l *winEventLogExp) openFile(state checkpoint.EventLogState, bookmark win.Bookmark) (win.EvtHandle, error) { + path := l.channelName + + h, err := win.EvtQuery(0, path, "", win.EvtQueryFilePath|win.EvtQueryForwardDirection) + if err != nil { + return win.NilHandle, errors.Wrapf(err, "failed to get handle to event log file %v", path) + } + + if bookmark > 0 { + l.log.Debugf("Seeking to bookmark. timestamp=%v bookmark=%v", + state.Timestamp, state.Bookmark) + + // This seeks to the last read event and strictly validates that the + // bookmarked record number exists. + if err = win.EvtSeek(h, 0, win.EvtHandle(bookmark), win.EvtSeekRelativeToBookmark|win.EvtSeekStrict); err == nil { + // Then we advance past the last read event to avoid sending that + // event again. This won't fail if we're at the end of the file. + err = errors.Wrap( + win.EvtSeek(h, 1, win.EvtHandle(bookmark), win.EvtSeekRelativeToBookmark), + "failed to seek past bookmarked position") + } else { + l.log.Warnf("s Failed to seek to bookmarked location in %v (error: %v). "+ + "Recovering by reading the log from the beginning. (Did the file "+ + "change since it was last read?)", path, err) + err = errors.Wrap( + win.EvtSeek(h, 0, 0, win.EvtSeekRelativeToFirst), + "failed to seek to beginning of log") + } + + if err != nil { + return win.NilHandle, err + } + } + + return h, err +} + +func (l *winEventLogExp) Read() ([]Record, error) { + var records []Record + + for h, ok := l.iterator.Next(); ok; h, ok = l.iterator.Next() { + record, err := l.processHandle(h) + if err != nil { + l.log.Warnw("Dropping event due to rendering error.", "error", err) + incrementMetric(dropReasons, err) + continue + } + records = append(records, *record) + + // It has read the maximum requested number of events. + if len(records) >= l.maxRead { + return records, nil + } + } + + // An error occurred while retrieving more events. + if err := l.iterator.Err(); err != nil { + return records, err + } + + // Reader is configured to stop when there are no more events. + if Stop == l.config.NoMoreEvents { + return records, io.EOF + } + + return records, nil +} + +func (l *winEventLogExp) processHandle(h win.EvtHandle) (*Record, error) { + defer h.Close() + + // NOTE: Render can return an error and a partial event. + evt, err := l.renderer.Render(h) + if evt == nil { + return nil, err + } + if err != nil { + evt.RenderErr = append(evt.RenderErr, err.Error()) + } + + // TODO: Need to add XML when configured. + + r := &Record{ + API: winEventLogExpAPIName, + Event: *evt, + } + + if l.file { + r.File = l.channelName + } + + r.Offset = checkpoint.EventLogState{ + Name: l.channelName, + RecordNumber: r.RecordID, + Timestamp: r.TimeCreated.SystemTime, + } + if r.Offset.Bookmark, err = l.createBookmarkFromEvent(h); err != nil { + l.log.Warnw("Failed creating bookmark.", "error", err) + } + l.lastRead = r.Offset + return r, nil +} + +func (l *winEventLogExp) createBookmarkFromEvent(evtHandle win.EvtHandle) (string, error) { + bookmark, err := win.NewBookmarkFromEvent(evtHandle) + if err != nil { + return "", errors.Wrap(err, "failed to create new bookmark from event handle") + } + defer bookmark.Close() + + return bookmark.XML() +} + +func (l *winEventLogExp) Close() error { + l.log.Debug("Closing event log reader handles.") + return multierr.Combine( + l.iterator.Close(), + l.renderer.Close(), + ) +} + +// newWinEventLogExp creates and returns a new EventLog for reading event logs +// using the Windows Event Log. +func newWinEventLogExp(options *common.Config) (EventLog, error) { + cfgwarn.Experimental("The %s event log reader is experimental.", winEventLogExpAPIName) + + c := winEventLogConfig{BatchReadSize: 512} + if err := readConfig(options, &c, winEventLogConfigKeys); err != nil { + return nil, err + } + + queryLog := c.Name + isFile := false + if info, err := os.Stat(c.Name); err == nil && info.Mode().IsRegular() { + path, err := filepath.Abs(c.Name) + if err != nil { + return nil, err + } + isFile = true + queryLog = "file://" + path + } + + query, err := win.Query{ + Log: queryLog, + IgnoreOlder: c.SimpleQuery.IgnoreOlder, + Level: c.SimpleQuery.Level, + EventID: c.SimpleQuery.EventID, + Provider: c.SimpleQuery.Provider, + }.Build() + if err != nil { + return nil, err + } + + log := logp.NewLogger("wineventlog").With("channel", c.Name) + + renderer, err := win.NewRenderer(win.NilHandle, log) + if err != nil { + return nil, err + } + + l := &winEventLogExp{ + config: c, + query: query, + channelName: c.Name, + file: isFile, + maxRead: c.BatchReadSize, + renderer: renderer, + log: log, + } + + return l, nil +} + +func init() { + // Register wineventlog API if it is available. + available, _ := win.IsAvailable() + if available { + Register(winEventLogExpAPIName, 10, newWinEventLogExp, win.Channels) + } +} diff --git a/winlogbeat/eventlog/wineventlog_test.go b/winlogbeat/eventlog/wineventlog_test.go index c37615551a7..a9205113f78 100644 --- a/winlogbeat/eventlog/wineventlog_test.go +++ b/winlogbeat/eventlog/wineventlog_test.go @@ -20,122 +20,191 @@ package eventlog import ( - "expvar" + "io" + "os/exec" "path/filepath" "strconv" + "strings" "testing" - elog "github.com/andrewkroh/sys/windows/svc/eventlog" + "github.com/andrewkroh/sys/windows/svc/eventlog" "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/winlogbeat/checkpoint" + "github.com/elastic/beats/v7/winlogbeat/sys/wineventlog" ) -func TestWinEventLogBatchReadSize(t *testing.T) { - configureLogp() - log, err := initLog(providerName, sourceName, eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) - if err != nil { - t.Fatal(err) - } - }() +func TestWindowsEventLogAPI(t *testing.T) { + testWindowsEventLog(t, winEventLogAPIName) +} - // Publish test messages: - for k, m := range messages { - err = log.Report(m.eventType, k, []string{m.message}) +func TestWindowsEventLogAPIExperimental(t *testing.T) { + testWindowsEventLog(t, winEventLogExpAPIName) +} + +func testWindowsEventLog(t *testing.T, api string) { + writer, teardown := createLog(t) + defer teardown() + + setLogSize(t, providerName, gigabyte) + + // Publish large test messages. + const totalEvents = 1000 + for i := 0; i < totalEvents; i++ { + err := writer.Report(eventlog.Info, uint32(i%1000), []string{strconv.Itoa(i) + " " + randomSentence(31800)}) if err != nil { t.Fatal(err) } } - batchReadSize := 2 - eventlog, teardown := setupWinEventLog(t, 0, map[string]interface{}{"name": providerName, "batch_read_size": batchReadSize}) - defer teardown() - - records, err := eventlog.Read() - if err != nil { - t.Fatal(err) + openLog := func(t testing.TB, config map[string]interface{}) EventLog { + return openLog(t, api, nil, config) } - assert.Len(t, records, batchReadSize) -} + t.Run("batch_read_size_config", func(t *testing.T) { + const batchReadSize = 2 -// TestReadLargeBatchSize tests reading from an event log using a large -// read_batch_size parameter. When combined with large messages this causes -// EvtNext (wineventlog.EventRecords) to fail with RPC_S_INVALID_BOUND error. -func TestReadLargeBatchSize(t *testing.T) { - configureLogp() - log, err := initLog(providerName, sourceName, eventCreateMsgFile) - if err != nil { - t.Fatal(err) - } - defer func() { - err := uninstallLog(providerName, sourceName, log) + log := openLog(t, map[string]interface{}{"name": providerName, "batch_read_size": batchReadSize}) + defer log.Close() + + records, err := log.Read() if err != nil { t.Fatal(err) } - }() - setLogSize(t, providerName, gigabyte) + assert.Len(t, records, batchReadSize) + }) - // Publish large test messages. - totalEvents := 1000 - for i := 0; i < totalEvents; i++ { - err = log.Report(elog.Info, uint32(i%1000), []string{strconv.Itoa(i) + " " + randomSentence(31800)}) - if err != nil { - t.Fatal("ReportEvent error", err) + // Test reading from an event log using a large batch_read_size parameter. + // When combined with large messages this causes EvtNext to fail with + // RPC_S_INVALID_BOUND error. The reader should recover from the error. + t.Run("large_batch_read", func(t *testing.T) { + log := openLog(t, map[string]interface{}{"name": providerName, "batch_read_size": 1024}) + defer log.Close() + + var eventCount int + + for eventCount < totalEvents { + records, err := log.Read() + if err != nil { + t.Fatal("read error", err) + } + if len(records) == 0 { + t.Fatal("read returned 0 records") + } + + t.Logf("Read() returned %d events.", len(records)) + eventCount += len(records) } - } - eventlog, teardown := setupWinEventLog(t, 0, map[string]interface{}{"name": providerName, "batch_read_size": 1024}) - defer teardown() + assert.Equal(t, totalEvents, eventCount) + }) - var eventCount int - for eventCount < totalEvents { - records, err := eventlog.Read() + t.Run("evtx_file", func(t *testing.T) { + path, err := filepath.Abs("../sys/wineventlog/testdata/sysmon-9.01.evtx") if err != nil { - t.Fatal("read error", err) - } - if len(records) == 0 { - t.Fatal("read returned 0 records") + t.Fatal(err) } - eventCount += len(records) - } - t.Logf("number of records returned: %v", eventCount) + log := openLog(t, map[string]interface{}{ + "name": path, + "no_more_events": "stop", + }) + defer log.Close() - wineventlog := eventlog.(*winEventLog) - assert.Equal(t, 1024, wineventlog.maxRead) + records, err := log.Read() + + // This implementation returns the EOF on the next call. + if err == nil && api == winEventLogAPIName { + _, err = log.Read() + } - expvar.Do(func(kv expvar.KeyValue) { - if kv.Key == "read_errors" { - t.Log(kv) + if assert.Error(t, err, "no_more_events=stop requires io.EOF to be returned") { + assert.Equal(t, io.EOF, err) } + + assert.Len(t, records, 32) }) } -func TestReadEvtxFile(t *testing.T) { - path, err := filepath.Abs("../sys/wineventlog/testdata/sysmon-9.01.evtx") +// ---- Utility Functions ----- + +// createLog creates a new event log and returns a handle for writing events +// to the log. +func createLog(t testing.TB, messageFiles ...string) (log *eventlog.Log, tearDown func()) { + const name = providerName + const source = sourceName + + messageFile := eventCreateMsgFile + if len(messageFiles) > 0 { + messageFile = strings.Join(messageFiles, ";") + } + + existed, err := eventlog.Install(name, source, messageFile, true, eventlog.Error|eventlog.Warning|eventlog.Info) if err != nil { t.Fatal(err) } - configureLogp() - eventlog, teardown := setupWinEventLog(t, 0, map[string]interface{}{ - "name": path, - }) - defer teardown() + if existed { + wineventlog.EvtClearLog(wineventlog.NilHandle, name, "") + } - records, err := eventlog.Read() + log, err = eventlog.Open(source) if err != nil { + eventlog.RemoveSource(name, source) + eventlog.RemoveProvider(name) t.Fatal(err) } - assert.Len(t, records, 32) + tearDown = func() { + log.Close() + wineventlog.EvtClearLog(wineventlog.NilHandle, name, "") + eventlog.RemoveSource(name, source) + eventlog.RemoveProvider(name) + } + + return log, tearDown } -func setupWinEventLog(t *testing.T, recordID uint64, options map[string]interface{}) (EventLog, func()) { - return setupEventLog(t, newWinEventLog, recordID, options) +// setLogSize set the maximum number of bytes that an event log can hold. +func setLogSize(t testing.TB, provider string, sizeBytes int) { + output, err := exec.Command("wevtutil.exe", "sl", "/ms:"+strconv.Itoa(sizeBytes), provider).CombinedOutput() + if err != nil { + t.Fatal("Failed to set log size", err, string(output)) + } +} + +func openLog(t testing.TB, api string, state *checkpoint.EventLogState, config map[string]interface{}) EventLog { + cfg, err := common.NewConfigFrom(config) + if err != nil { + t.Fatal(err) + } + + var log EventLog + switch api { + case winEventLogAPIName: + log, err = newWinEventLog(cfg) + case winEventLogExpAPIName: + log, err = newWinEventLogExp(cfg) + case eventLoggingAPIName: + log, err = newEventLogging(cfg) + default: + t.Fatalf("Unknown API name: '%s'", api) + } + if err != nil { + t.Fatal(err) + } + + var eventLogState checkpoint.EventLogState + if state != nil { + eventLogState = *state + } + + if err = log.Open(eventLogState); err != nil { + log.Close() + t.Fatal(err) + } + + return log } diff --git a/winlogbeat/sys/event.go b/winlogbeat/sys/event.go index 50df9ec18d3..d88617d8925 100644 --- a/winlogbeat/sys/event.go +++ b/winlogbeat/sys/event.go @@ -41,6 +41,7 @@ type Event struct { LevelRaw uint8 `xml:"System>Level"` TaskRaw uint16 `xml:"System>Task"` OpcodeRaw uint8 `xml:"System>Opcode"` + KeywordsRaw HexInt64 `xml:"System>Keywords"` TimeCreated TimeCreated `xml:"System>TimeCreated"` RecordID uint64 `xml:"System>EventRecordID"` Correlation Correlation `xml:"System>Correlation"` @@ -96,7 +97,7 @@ type Execution struct { ProcessorTime uint32 `xml:"ProcessorTime,attr"` } -// EventIdentifier is the identifer that the provider uses to identify a +// EventIdentifier is the identifier that the provider uses to identify a // specific event type. type EventIdentifier struct { Qualifiers uint16 `xml:"Qualifiers,attr"` @@ -225,3 +226,21 @@ func (v *Version) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { *v = Version(version) return nil } + +type HexInt64 uint64 + +func (v *HexInt64) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var s string + if err := d.DecodeElement(&s, &start); err != nil { + return err + } + + num, err := strconv.ParseInt(s, 0, 64) + if err != nil { + // Ignore invalid version values. + return nil + } + + *v = HexInt64(num) + return nil +} diff --git a/winlogbeat/sys/event_test.go b/winlogbeat/sys/event_test.go index 0684e99b473..8d0f6ee04f8 100644 --- a/winlogbeat/sys/event_test.go +++ b/winlogbeat/sys/event_test.go @@ -94,6 +94,7 @@ func TestXML(t *testing.T) { EventIdentifier: EventIdentifier{ID: 91}, LevelRaw: 4, TaskRaw: 9, + KeywordsRaw: 0x4000000000000004, TimeCreated: TimeCreated{allXMLTimeCreated}, RecordID: 100, Correlation: Correlation{"{A066CCF1-8AB3-459B-B62F-F79F957A5036}", "{85FC0930-9C49-42DA-804B-A7368104BD1B}"}, diff --git a/winlogbeat/sys/wineventlog/bookmark.go b/winlogbeat/sys/wineventlog/bookmark.go new file mode 100644 index 00000000000..fa806aa2c34 --- /dev/null +++ b/winlogbeat/sys/wineventlog/bookmark.go @@ -0,0 +1,81 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "syscall" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +// Bookmark is a handle to an event log bookmark. +type Bookmark EvtHandle + +// Close closes the bookmark handle. +func (b Bookmark) Close() error { + return EvtHandle(b).Close() +} + +// XML returns the bookmark's value as XML. +func (b Bookmark) XML() (string, error) { + var bufferUsed uint32 + + err := _EvtRender(NilHandle, EvtHandle(b), EvtRenderBookmark, 0, nil, &bufferUsed, nil) + if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { + return "", errors.Wrap(err, "failed to determine necessary buffer size for EvtRender") + } + + bb := newByteBuffer() + bb.Reserve(int(bufferUsed * 2)) + defer bb.free() + + err = _EvtRender(NilHandle, EvtHandle(b), EvtRenderBookmark, uint32(len(bb.buf)), &bb.buf[0], &bufferUsed, nil) + if err != nil { + return "", errors.Wrap(err, "failed to render bookmark XML") + } + + return UTF16BytesToString(bb.buf) +} + +// NewBookmarkFromEvent returns a Bookmark pointing to the given event record. +// The returned handle must be closed. +func NewBookmarkFromEvent(eventHandle EvtHandle) (Bookmark, error) { + h, err := _EvtCreateBookmark(nil) + if err != nil { + return 0, err + } + if err = _EvtUpdateBookmark(h, eventHandle); err != nil { + h.Close() + return 0, err + } + return Bookmark(h), nil +} + +// NewBookmarkFromXML returns a Bookmark created from an XML bookmark. +// The returned handle must be closed. +func NewBookmarkFromXML(xml string) (Bookmark, error) { + utf16, err := syscall.UTF16PtrFromString(xml) + if err != nil { + return 0, err + } + h, err := _EvtCreateBookmark(utf16) + return Bookmark(h), err +} diff --git a/winlogbeat/sys/wineventlog/bookmark_test.go b/winlogbeat/sys/wineventlog/bookmark_test.go new file mode 100644 index 00000000000..34a443a4184 --- /dev/null +++ b/winlogbeat/sys/wineventlog/bookmark_test.go @@ -0,0 +1,88 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBookmark(t *testing.T) { + log := openLog(t, security4752File) + defer log.Close() + + evtHandle := mustNextHandle(t, log) + defer evtHandle.Close() + + t.Run("NewBookmarkFromEvent", func(t *testing.T) { + bookmark, err := NewBookmarkFromEvent(evtHandle) + if err != nil { + t.Fatal(err) + } + defer func() { + assert.NoError(t, bookmark.Close()) + }() + + xml, err := bookmark.XML() + if err != nil { + t.Fatal(err) + } + + assert.Contains(t, xml, "") + }) + + t.Run("NewBookmarkFromXML", func(t *testing.T) { + const savedBookmarkXML = ` + + +` + + bookmark, err := NewBookmarkFromXML(savedBookmarkXML) + if err != nil { + t.Fatal(err) + } + defer func() { + assert.NoError(t, bookmark.Close()) + }() + + xml, err := bookmark.XML() + if err != nil { + t.Fatal(err) + } + + // Ignore whitespace differences. + normalizer := strings.NewReplacer(" ", "", "\r\n", "", "\n", "") + assert.Equal(t, normalizer.Replace(savedBookmarkXML), normalizer.Replace(xml)) + }) + + t.Run("NewBookmarkFromEvent_invalid", func(t *testing.T) { + bookmark, err := NewBookmarkFromEvent(NilHandle) + assert.Error(t, err) + assert.Zero(t, bookmark) + }) + + t.Run("NewBookmarkFromXML_invalid", func(t *testing.T) { + bookmark, err := NewBookmarkFromXML("{Not XML}") + assert.Error(t, err) + assert.Zero(t, bookmark) + }) +} diff --git a/winlogbeat/sys/wineventlog/bufferpool.go b/winlogbeat/sys/wineventlog/bufferpool.go new file mode 100644 index 00000000000..104d45f938a --- /dev/null +++ b/winlogbeat/sys/wineventlog/bufferpool.go @@ -0,0 +1,113 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package wineventlog + +import ( + "sync" + + "github.com/elastic/beats/v7/winlogbeat/sys" +) + +// bufferPool contains a pool of byteBuffer objects. +var bufferPool = sync.Pool{ + New: func() interface{} { return &byteBuffer{buf: make([]byte, 1024)} }, +} + +// byteBuffer is an expandable buffer backed by a byte slice. +type byteBuffer struct { + buf []byte + offset int +} + +// newByteBuffer return a byteBuffer from the pool. The returned value must +// be released with free(). +func newByteBuffer() *byteBuffer { + b := bufferPool.Get().(*byteBuffer) + b.Reset() + return b +} + +// free returns the byteBuffer to the pool. +func (b *byteBuffer) free() { + if b == nil { + return + } + bufferPool.Put(b) +} + +// Write appends the contents of p to the buffer, growing the buffer as needed. +// The return value is the length of p; err is always nil. This implements +// io.Writer. +func (b *byteBuffer) Write(p []byte) (int, error) { + if len(b.buf) < b.offset+len(p) { + // Create a buffer larger than needed so we don't spend lots of time + // allocating and copying. + spaceNeeded := len(b.buf) - b.offset + len(p) + largerBuf := make([]byte, 2*len(b.buf)+spaceNeeded) + copy(largerBuf, b.buf[:b.offset]) + b.buf = largerBuf + } + n := copy(b.buf[b.offset:], p) + b.offset += n + return n, nil +} + +// Reset resets the buffer to be empty. It retains the same underlying storage +// capacity. +func (b *byteBuffer) Reset() { + b.offset = 0 + b.buf = b.buf[:cap(b.buf)] +} + +// Bytes returns a slice of length b.Len() holding the bytes that have been +// written to the buffer. +func (b *byteBuffer) Bytes() []byte { + return b.buf[:b.offset] +} + +// Len returns the number of bytes that have been written to the buffer. +func (b *byteBuffer) Len() int { + return b.offset +} + +// Reserve reserves n bytes by increasing the buffer's length. It may allocate +// a new underlying buffer discarding any existing contents. +func (b *byteBuffer) Reserve(n int) { + b.offset = n + + if n > cap(b.buf) { + // Allocate new larger buffer with len=n. + b.buf = make([]byte, n) + } else { + b.buf = b.buf[:n] + } +} + +// UTF16BytesToString converts the given UTF-16 bytes to a string. +func UTF16BytesToString(b []byte) (string, error) { + // Use space from the byteBuffer pool as working memory for the conversion. + bb := newByteBuffer() + defer bb.free() + + if err := sys.UTF16ToUTF8Bytes(b, bb); err != nil { + return "", err + } + + // This copies the UTF-8 bytes to create a string. + return string(bb.Bytes()), nil +} diff --git a/winlogbeat/sys/wineventlog/doc.go b/winlogbeat/sys/wineventlog/doc.go index 7c3936a0fb7..09c8685c331 100644 --- a/winlogbeat/sys/wineventlog/doc.go +++ b/winlogbeat/sys/wineventlog/doc.go @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. -/* -Package wineventlog provides access to the Windows Event Log API used in -all versions of Windows since Vista (i.e. Windows 7+ and Windows Server 2008+). -This is distinct from the Event Logging API that was used in Windows XP, -Windows Server 2003, and Windows 2000. -*/ +// Package wineventlog provides access to the Windows Event Log API used in +// all versions of Windows since Vista (i.e. Windows 7+ and Windows Server 2008+). +// This is distinct from the Event Logging API that was used in Windows XP, +// Windows Server 2003, and Windows 2000. package wineventlog + +// Add -trace to enable debug prints around syscalls. +//go:generate go get golang.org/x/sys/windows/mkwinsyscall +//go:generate $GOPATH/bin/mkwinsyscall.exe -systemdll -output zsyscall_windows.go syscall_windows.go diff --git a/winlogbeat/sys/wineventlog/format_message.go b/winlogbeat/sys/wineventlog/format_message.go new file mode 100644 index 00000000000..e8befcdeae3 --- /dev/null +++ b/winlogbeat/sys/wineventlog/format_message.go @@ -0,0 +1,102 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "unsafe" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +// getMessageStringFromHandle returns the message for the given eventHandle. +func getMessageStringFromHandle(metadata *PublisherMetadata, eventHandle EvtHandle, values []EvtVariant) (string, error) { + return getMessageString(metadata, eventHandle, 0, values) +} + +// getMessageStringFromMessageID returns the message associated with the given +// message ID. +func getMessageStringFromMessageID(metadata *PublisherMetadata, messageID uint32, values []EvtVariant) (string, error) { + return getMessageString(metadata, NilHandle, messageID, values) +} + +// getMessageString returns an event's message. Don't use this directly. Instead +// use either getMessageStringFromHandle or getMessageStringFromMessageID. +func getMessageString(metadata *PublisherMetadata, eventHandle EvtHandle, messageID uint32, values []EvtVariant) (string, error) { + var flags EvtFormatMessageFlag + if eventHandle > 0 { + flags = EvtFormatMessageEvent + } else if messageID > 0 { + flags = EvtFormatMessageId + } + + metadataHandle := NilHandle + if metadata != nil { + metadataHandle = metadata.Handle + } + + return evtFormatMessage(metadataHandle, eventHandle, messageID, values, flags) +} + +// getEventXML returns all data in the event as XML. +func getEventXML(metadata *PublisherMetadata, eventHandle EvtHandle) (string, error) { + metadataHandle := NilHandle + if metadata != nil { + metadataHandle = metadata.Handle + } + return evtFormatMessage(metadataHandle, eventHandle, 0, nil, EvtFormatMessageXml) +} + +// evtFormatMessage uses EvtFormatMessage to generate a string. +func evtFormatMessage(metadataHandle EvtHandle, eventHandle EvtHandle, messageID uint32, values []EvtVariant, messageFlag EvtFormatMessageFlag) (string, error) { + var ( + valuesCount = uint32(len(values)) + valuesPtr uintptr + ) + if len(values) > 0 { + valuesPtr = uintptr(unsafe.Pointer(&values[0])) + } + + // Determine the buffer size needed (given in WCHARs). + var bufferUsed uint32 + err := _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, 0, nil, &bufferUsed) + if err != windows.ERROR_INSUFFICIENT_BUFFER { + return "", errors.Wrap(err, "failed in EvtFormatMessage") + } + + // Get a buffer from the pool and adjust its length. + bb := newByteBuffer() + defer bb.free() + bb.Reserve(int(bufferUsed * 2)) + + err = _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, uint32(len(bb.buf)/2), &bb.buf[0], &bufferUsed) + if err != nil { + switch err { + // Ignore some errors so it can tolerate missing or mismatched parameter values. + case windows.ERROR_EVT_UNRESOLVED_VALUE_INSERT: + case windows.ERROR_EVT_UNRESOLVED_PARAMETER_INSERT: + case windows.ERROR_EVT_MAX_INSERTS_REACHED: + default: + return "", errors.Wrap(err, "failed in EvtFormatMessage") + } + } + + return UTF16BytesToString(bb.buf) +} diff --git a/winlogbeat/sys/wineventlog/format_message_test.go b/winlogbeat/sys/wineventlog/format_message_test.go new file mode 100644 index 00000000000..492061f7a28 --- /dev/null +++ b/winlogbeat/sys/wineventlog/format_message_test.go @@ -0,0 +1,153 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFormatMessage(t *testing.T) { + log := openLog(t, security4752File) + defer log.Close() + + evtHandle := mustNextHandle(t, log) + defer evtHandle.Close() + + publisherMetadata, err := NewPublisherMetadata(NilHandle, "Microsoft-Windows-Security-Auditing") + if err != nil { + t.Fatal(err) + } + defer publisherMetadata.Close() + + t.Run("getMessageStringFromHandle", func(t *testing.T) { + t.Run("no_metadata", func(t *testing.T) { + // Metadata is required unless the events were forwarded with "RenderedText". + _, err := getMessageStringFromHandle(nil, evtHandle, nil) + assert.Error(t, err) + }) + + t.Run("with_metadata", func(t *testing.T) { + // When no values are passed in then event data from the event is + // substituted into the message. + msg, err := getMessageStringFromHandle(publisherMetadata, evtHandle, nil) + if err != nil { + t.Fatal(err) + } + assert.Contains(t, msg, "CN=Administrator,CN=Users,DC=TEST,DC=SAAS") + }) + + t.Run("custom_values", func(t *testing.T) { + // Substitute custom values into the message. + msg, err := getMessageStringFromHandle(publisherMetadata, evtHandle, templateInserts.Slice()) + if err != nil { + t.Fatal(err) + } + + assert.Contains(t, msg, `{{eventParam $ 2}}`) + + // NOTE: In this test case I noticed the messages contains + // "Logon ID: 0x0" + // but it should contain + // "Logon ID: {{eventParam $ 9}}" + // + // This may mean that certain windows.GUID values cannot be + // substituted with string values. So we shouldn't rely on this + // method to create text/templates. Instead we can use the + // getMessageStringFromMessageID (see test below) that works as + // expected. + assert.NotContains(t, msg, `{{eventParam $ 9}}`) + }) + }) + + t.Run("getMessageStringFromMessageID", func(t *testing.T) { + // Get the message ID for event 4752. + itr, err := publisherMetadata.EventMetadataIterator() + if err != nil { + t.Fatal(err) + } + defer itr.Close() + + var messageID uint32 + for itr.Next() { + id, err := itr.EventID() + if err != nil { + t.Fatal(err) + } + if id == 4752 { + messageID, err = itr.MessageID() + if err != nil { + t.Fatal(err) + } + } + } + + if messageID == 0 { + t.Fatal("message ID for event 4752 not found") + } + + t.Run("no_metadata", func(t *testing.T) { + // Metadata is required to find the message file. + _, err := getMessageStringFromMessageID(nil, messageID, nil) + assert.Error(t, err) + }) + + t.Run("with_metadata", func(t *testing.T) { + // When no values are passed in then the raw message is returned + // with place-holders like %1 and %2. + msg, err := getMessageStringFromMessageID(publisherMetadata, messageID, nil) + if err != nil { + t.Fatal(err) + } + + assert.Contains(t, msg, "%9") + }) + + t.Run("custom_values", func(t *testing.T) { + msg, err := getMessageStringFromMessageID(publisherMetadata, messageID, templateInserts.Slice()) + if err != nil { + t.Fatal(err) + } + + assert.Contains(t, msg, `{{eventParam $ 2}}`) + assert.Contains(t, msg, `{{eventParam $ 9}}`) + }) + }) + + t.Run("getEventXML", func(t *testing.T) { + t.Run("no_metadata", func(t *testing.T) { + // It needs the metadata handle to add the message to the XML. + _, err := getEventXML(nil, evtHandle) + assert.Error(t, err) + }) + + t.Run("with_metadata", func(t *testing.T) { + xml, err := getEventXML(publisherMetadata, evtHandle) + if err != nil { + t.Fatal(err) + } + + assert.True(t, strings.HasPrefix(xml, "")) + }) + }) +} diff --git a/winlogbeat/sys/wineventlog/iterator.go b/winlogbeat/sys/wineventlog/iterator.go new file mode 100644 index 00000000000..26492fe96b0 --- /dev/null +++ b/winlogbeat/sys/wineventlog/iterator.go @@ -0,0 +1,207 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "sync" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +const ( + evtNextMaxHandles = 1024 + evtNextDefaultHandles = 512 +) + +// EventIterator provides an iterator to read events from a log. It takes the +// place of calling EvtNext directly. +type EventIterator struct { + subscriptionFactory SubscriptionFactory // Factory for producing a new subscription handle. + subscription EvtHandle // Handle from EvtQuery or EvtSubscribe. + batchSize uint32 // Number of handles to request by default. + handles [evtNextMaxHandles]EvtHandle // Handles returned by EvtNext. + lastErr error // Last error returned by EvtNext. + active []EvtHandle // Slice of the handles array containing the valid unread handles. + mutex sync.Mutex // Mutex to enable parallel iteration. + + // For testing purposes to be able to mock EvtNext. + evtNext func(resultSet EvtHandle, eventArraySize uint32, eventArray *EvtHandle, timeout uint32, flags uint32, numReturned *uint32) (err error) +} + +// SubscriptionFactory produces a handle from EvtQuery or EvtSubscribe that +// points to the next unread event. Provide a factory to enable automatic +// recover of certain errors. +type SubscriptionFactory func() (EvtHandle, error) + +// EventIteratorOption represents a configuration of for the construction of +// the EventIterator. +type EventIteratorOption func(*EventIterator) + +// WithSubscriptionFactory configures a SubscriptionFactory for the iterator to +// use to create a subscription handle. +func WithSubscriptionFactory(factory SubscriptionFactory) EventIteratorOption { + return func(itr *EventIterator) { + itr.subscriptionFactory = factory + } +} + +// WithSubscription configures the iterator with an existing subscription handle. +func WithSubscription(subscription EvtHandle) EventIteratorOption { + return func(itr *EventIterator) { + itr.subscription = subscription + } +} + +// WithBatchSize configures the number of handles the iterator will request +// when calling EvtNext. Valid batch sizes range on [1, 1024]. +func WithBatchSize(size int) EventIteratorOption { + return func(itr *EventIterator) { + if size > 0 { + itr.batchSize = uint32(size) + } + if size > evtNextMaxHandles { + itr.batchSize = evtNextMaxHandles + } + } +} + +// NewEventIterator creates an iterator to read event handles from a subscription. +// The iterator is thread-safe. +func NewEventIterator(opts ...EventIteratorOption) (*EventIterator, error) { + itr := &EventIterator{ + batchSize: evtNextDefaultHandles, + evtNext: _EvtNext, + } + + for _, opt := range opts { + opt(itr) + } + + if itr.subscriptionFactory == nil && itr.subscription == NilHandle { + return nil, errors.New("either a subscription or subscription factory is required") + } + + if itr.subscription == NilHandle { + handle, err := itr.subscriptionFactory() + if err != nil { + return nil, err + } + itr.subscription = handle + } + + return itr, nil +} + +// Next advances the iterator to the next handle. After Next returns false, the +// Err() method will return any error that occurred during iteration, except +// that if it was windows.ERROR_NO_MORE_ITEMS, Err() will return nil and you +// may call Next() again later to check if new events are available. +func (itr *EventIterator) Next() (EvtHandle, bool) { + itr.mutex.Lock() + defer itr.mutex.Unlock() + + if itr.lastErr != nil { + return NilHandle, false + } + + if !itr.empty() { + itr.active = itr.active[1:] + } + + if itr.empty() && !itr.moreHandles() { + return NilHandle, false + } + + return itr.active[0], true +} + +// empty returns true when there are no more handles left to read from memory. +func (itr *EventIterator) empty() bool { + return len(itr.active) == 0 +} + +// moreHandles fetches more handles using EvtNext. It returns true if it +// successfully fetched more handles. +func (itr *EventIterator) moreHandles() bool { + batchSize := itr.batchSize + + for batchSize > 0 { + var numReturned uint32 + + err := itr.evtNext(itr.subscription, batchSize, &itr.handles[0], 0, 0, &numReturned) + switch err { + case nil: + itr.lastErr = nil + itr.active = itr.handles[:numReturned] + case windows.ERROR_NO_MORE_ITEMS, windows.ERROR_INVALID_OPERATION: + case windows.RPC_S_INVALID_BOUND: + // Attempt automated recovery if we have a factory. + if itr.subscriptionFactory != nil { + itr.subscription.Close() + itr.subscription, err = itr.subscriptionFactory() + if err != nil { + itr.lastErr = errors.Wrap(err, "failed in EvtNext while trying to "+ + "recover from RPC_S_INVALID_BOUND error") + return false + } + + // Reduce batch size and try again. + batchSize = batchSize / 2 + continue + } else { + itr.lastErr = errors.Wrap(err, "failed in EvtNext (try "+ + "reducing the batch size or providing a subscription "+ + "factory for automatic recovery)") + } + default: + itr.lastErr = err + } + + break + } + + return !itr.empty() +} + +// Err returns the first non-ERROR_NO_MORE_ITEMS error encountered by the +// EventIterator. +// +// Some Windows versions will fail with windows.RPC_S_INVALID_BOUND when the +// batch size is too large. If this occurs you can recover by closing the +// iterator, creating a new subscription, seeking to the next unread event, and +// creating a new EventIterator with a smaller batch size. +func (itr *EventIterator) Err() error { + itr.mutex.Lock() + defer itr.mutex.Unlock() + + return itr.lastErr +} + +// Close closes the subscription handle and any unread event handles. +func (itr *EventIterator) Close() error { + itr.mutex.Lock() + defer itr.mutex.Unlock() + + for _, h := range itr.active { + h.Close() + } + return itr.subscription.Close() +} diff --git a/winlogbeat/sys/wineventlog/iterator_test.go b/winlogbeat/sys/wineventlog/iterator_test.go new file mode 100644 index 00000000000..fee6553f36c --- /dev/null +++ b/winlogbeat/sys/wineventlog/iterator_test.go @@ -0,0 +1,271 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "strconv" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "golang.org/x/sys/windows" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestEventIterator(t *testing.T) { + logp.TestingSetup() + + writer, tearDown := createLog(t) + defer tearDown() + + const eventCount = 1500 + for i := 0; i < eventCount; i++ { + if err := writer.Info(1, "Test message "+strconv.Itoa(i+1)); err != nil { + t.Fatal(err) + } + } + + // Validate the assumption that 1024 is the max number of handles supported + // by EvtNext. + t.Run("max_handles_assumption", func(t *testing.T) { + log := openLog(t, winlogbeatTestLogName) + defer log.Close() + + var ( + numReturned uint32 + handles = [evtNextMaxHandles + 1]EvtHandle{} + ) + + // Too many handles. + err := _EvtNext(log, uint32(len(handles)), &handles[0], 0, 0, &numReturned) + assert.Equal(t, windows.ERROR_INVALID_PARAMETER, err) + + // The max number of handles. + err = _EvtNext(log, evtNextMaxHandles, &handles[0], 0, 0, &numReturned) + if assert.NoError(t, err) { + for _, h := range handles[:numReturned] { + h.Close() + } + } + }) + + t.Run("no_subscription", func(t *testing.T) { + _, err := NewEventIterator() + assert.Error(t, err) + }) + + t.Run("with_subscription", func(t *testing.T) { + log := openLog(t, winlogbeatTestLogName) + defer log.Close() + + itr, err := NewEventIterator(WithSubscription(log)) + if err != nil { + t.Fatal(err) + } + defer func() { assert.NoError(t, itr.Close()) }() + + assert.Nil(t, itr.subscriptionFactory) + assert.NotEqual(t, NilHandle, itr.subscription) + }) + + t.Run("with_subscription_factory", func(t *testing.T) { + factory := func() (handle EvtHandle, err error) { + return openLog(t, winlogbeatTestLogName), nil + } + itr, err := NewEventIterator(WithSubscriptionFactory(factory)) + if err != nil { + t.Fatal(err) + } + defer func() { assert.NoError(t, itr.Close()) }() + + assert.NotNil(t, itr.subscriptionFactory) + assert.NotEqual(t, NilHandle, itr.subscription) + }) + + t.Run("with_batch_size", func(t *testing.T) { + log := openLog(t, winlogbeatTestLogName) + defer log.Close() + + t.Run("default", func(t *testing.T) { + itr, err := NewEventIterator(WithSubscription(log)) + if err != nil { + t.Fatal(err) + } + assert.EqualValues(t, evtNextDefaultHandles, itr.batchSize) + }) + + t.Run("custom", func(t *testing.T) { + itr, err := NewEventIterator(WithSubscription(log), WithBatchSize(128)) + if err != nil { + t.Fatal(err) + } + assert.EqualValues(t, 128, itr.batchSize) + }) + + t.Run("too_small", func(t *testing.T) { + itr, err := NewEventIterator(WithSubscription(log), WithBatchSize(0)) + if err != nil { + t.Fatal(err) + } + assert.EqualValues(t, evtNextDefaultHandles, itr.batchSize) + }) + + t.Run("too_big", func(t *testing.T) { + itr, err := NewEventIterator(WithSubscription(log), WithBatchSize(evtNextMaxHandles+1)) + if err != nil { + t.Fatal(err) + } + assert.EqualValues(t, evtNextMaxHandles, itr.batchSize) + }) + }) + + t.Run("iterate", func(t *testing.T) { + log := openLog(t, winlogbeatTestLogName) + defer log.Close() + + itr, err := NewEventIterator(WithSubscription(log), WithBatchSize(13)) + if err != nil { + t.Fatal(err) + } + defer func() { assert.NoError(t, itr.Close()) }() + + var iterateCount int + for h, ok := itr.Next(); ok; h, ok = itr.Next() { + h.Close() + + if !assert.NotZero(t, h) { + return + } + + iterateCount++ + } + if err := itr.Err(); err != nil { + t.Fatal(err) + } + + assert.EqualValues(t, eventCount, iterateCount) + }) + + // Check for regressions of https://github.com/elastic/beats/issues/3076 + // where EvtNext fails reading batch of large events. + // + // Note: As of 2020-03 Windows 2019 no longer exhibits this behavior. + // Instead EvtNext simply returns fewer handles that the requested size. + t.Run("rpc_error", func(t *testing.T) { + log := openLog(t, winlogbeatTestLogName) + defer log.Close() + + // Mock the behavior to simplify testing since it's not reproducible + // on all Windows versions. + mockEvtNext := func(resultSet EvtHandle, eventArraySize uint32, eventArray *EvtHandle, timeout uint32, flags uint32, numReturned *uint32) (err error) { + if eventArraySize > 3 { + return windows.RPC_S_INVALID_BOUND + } + return _EvtNext(resultSet, eventArraySize, eventArray, timeout, flags, numReturned) + } + + // If you create the iterator with only a subscription handle then + // no recovery is possible without data loss. + t.Run("no_recovery", func(t *testing.T) { + itr, err := NewEventIterator(WithSubscription(log)) + if err != nil { + t.Fatal(err) + } + defer func() { assert.NoError(t, itr.Close()) }() + + itr.evtNext = mockEvtNext + + h, ok := itr.Next() + assert.False(t, ok) + assert.Zero(t, h) + if assert.Error(t, itr.Err()) { + assert.Contains(t, itr.Err().Error(), "try reducing the batch size") + assert.Equal(t, windows.RPC_S_INVALID_BOUND, errors.Cause(itr.Err())) + } + }) + + t.Run("automated_recovery", func(t *testing.T) { + var numFactoryInvocations int + var bookmark Bookmark + + // Create a proper subscription factor that resumes from the last + // read position by using bookmarks. + factory := func() (handle EvtHandle, err error) { + numFactoryInvocations++ + log := openLog(t, winlogbeatTestLogName) + + if bookmark != 0 { + // Seek to bookmark location. + err := EvtSeek(log, 0, EvtHandle(bookmark), EvtSeekRelativeToBookmark|EvtSeekStrict) + if err != nil { + t.Fatal(err) + } + + // Seek to one event after bookmark (unread position). + if err = EvtSeek(log, 1, NilHandle, EvtSeekRelativeToCurrent); err != nil { + t.Fatal(err) + } + } + + return log, err + } + + itr, err := NewEventIterator(WithSubscriptionFactory(factory), WithBatchSize(10)) + if err != nil { + t.Fatal(err) + } + defer func() { assert.NoError(t, itr.Close()) }() + + // Mock the EvtNext to cause the the RPC_S_INVALID_BOUND error. + itr.evtNext = mockEvtNext + + var iterateCount int + for h, ok := itr.Next(); ok; h, ok = itr.Next() { + func() { + defer h.Close() + + if !assert.NotZero(t, h) { + t.FailNow() + } + + // Store last read position. + if bookmark != 0 { + bookmark.Close() + } + bookmark, err = NewBookmarkFromEvent(h) + if err != nil { + t.Fatal(err) + } + + iterateCount++ + }() + } + if err := itr.Err(); err != nil { + t.Fatal(err) + } + + // Validate that the factory has been used to recover and + // that we received all the events. + assert.Greater(t, numFactoryInvocations, 1) + assert.EqualValues(t, eventCount, iterateCount) + }) + }) +} diff --git a/winlogbeat/sys/wineventlog/metadata_store.go b/winlogbeat/sys/wineventlog/metadata_store.go new file mode 100644 index 00000000000..e59294f6276 --- /dev/null +++ b/winlogbeat/sys/wineventlog/metadata_store.go @@ -0,0 +1,463 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "strconv" + "strings" + "sync" + "text/template" + + "github.com/pkg/errors" + "go.uber.org/multierr" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/winlogbeat/sys" +) + +var ( + // eventDataNameTransform removes spaces from parameter names. + eventDataNameTransform = strings.NewReplacer(" ", "_") + + // eventMessageTemplateFuncs contains functions for use in message templates. + eventMessageTemplateFuncs = template.FuncMap{ + "eventParam": eventParam, + } +) + +// publisherMetadataStore stores metadata from a publisher. +type publisherMetadataStore struct { + Metadata *PublisherMetadata // Handle to the publisher metadata. May be nil. + Keywords map[int64]string // Keyword bit mask to keyword name. + Opcodes map[uint8]string // Opcode value to name. + Levels map[uint8]string // Level value to name. + Tasks map[uint16]string // Task value to name. + + // Event ID to event metadata (message and event data param names). + Events map[uint16]*eventMetadata + // Event ID to map of fingerprints to event metadata. The fingerprint value + // is hash of the event data parameters count and types. + EventFingerprints map[uint16]map[uint64]*eventMetadata + + mutex sync.RWMutex + log *logp.Logger +} + +func newPublisherMetadataStore(session EvtHandle, provider string, log *logp.Logger) (*publisherMetadataStore, error) { + md, err := NewPublisherMetadata(session, provider) + if err != nil { + return nil, err + } + store := &publisherMetadataStore{ + Metadata: md, + EventFingerprints: map[uint16]map[uint64]*eventMetadata{}, + log: log.With("publisher", provider), + } + + // Query the provider metadata to build an in-memory cache of the + // information to optimize event reading. + err = multierr.Combine( + store.initKeywords(), + store.initOpcodes(), + store.initLevels(), + store.initTasks(), + store.initEvents(), + ) + if err != nil { + return nil, err + } + + return store, nil +} + +// newEmptyPublisherMetadataStore creates an empty metadata store for cases +// where no local publisher metadata exists. +func newEmptyPublisherMetadataStore(provider string, log *logp.Logger) *publisherMetadataStore { + return &publisherMetadataStore{ + Keywords: map[int64]string{}, + Opcodes: map[uint8]string{}, + Levels: map[uint8]string{}, + Tasks: map[uint16]string{}, + Events: map[uint16]*eventMetadata{}, + EventFingerprints: map[uint16]map[uint64]*eventMetadata{}, + log: log.With("publisher", provider, "empty", true), + } +} + +func (s *publisherMetadataStore) initKeywords() error { + keywords, err := s.Metadata.Keywords() + if err != nil { + return err + } + + s.Keywords = make(map[int64]string, len(keywords)) + for _, keywordMeta := range keywords { + val := keywordMeta.Name + if val == "" { + val = keywordMeta.Message + } + s.Keywords[int64(keywordMeta.Mask)] = val + } + return nil +} + +func (s *publisherMetadataStore) initOpcodes() error { + opcodes, err := s.Metadata.Opcodes() + if err != nil { + return err + } + s.Opcodes = make(map[uint8]string, len(opcodes)) + for _, opcodeMeta := range opcodes { + val := opcodeMeta.Message + if val == "" { + val = opcodeMeta.Name + } + s.Opcodes[uint8(opcodeMeta.Mask)] = val + } + return nil +} + +func (s *publisherMetadataStore) initLevels() error { + levels, err := s.Metadata.Levels() + if err != nil { + return err + } + + s.Levels = make(map[uint8]string, len(levels)) + for _, levelMeta := range levels { + val := levelMeta.Name + if val == "" { + val = levelMeta.Message + } + s.Levels[uint8(levelMeta.Mask)] = val + } + return nil +} + +func (s *publisherMetadataStore) initTasks() error { + tasks, err := s.Metadata.Tasks() + if err != nil { + return err + } + s.Tasks = make(map[uint16]string, len(tasks)) + for _, taskMeta := range tasks { + val := taskMeta.Message + if val == "" { + val = taskMeta.Name + } + s.Tasks[uint16(taskMeta.Mask)] = val + } + return nil +} + +func (s *publisherMetadataStore) initEvents() error { + itr, err := s.Metadata.EventMetadataIterator() + if err != nil { + return err + } + defer itr.Close() + + s.Events = map[uint16]*eventMetadata{} + for itr.Next() { + evt, err := newEventMetadataFromPublisherMetadata(itr, s.Metadata) + if err != nil { + s.log.Warnw("Failed to read event metadata from publisher. Continuing to next event.", + "error", err) + continue + } + s.Events[evt.EventID] = evt + } + return itr.Err() +} + +func (s *publisherMetadataStore) getEventMetadata(eventID uint16, eventDataFingerprint uint64, eventHandle EvtHandle) *eventMetadata { + // Use a read lock to get a cached value. + s.mutex.RLock() + fingerprints, found := s.EventFingerprints[eventID] + if found { + em, found := fingerprints[eventDataFingerprint] + if found { + s.mutex.RUnlock() + return em + } + } + + // Elevate to write lock. + s.mutex.RUnlock() + s.mutex.Lock() + defer s.mutex.Unlock() + + fingerprints, found = s.EventFingerprints[eventID] + if !found { + fingerprints = map[uint64]*eventMetadata{} + s.EventFingerprints[eventID] = fingerprints + } + + em, found := fingerprints[eventDataFingerprint] + if found { + return em + } + + // To ensure we always match the correct event data parameter names to + // values we will rely a fingerprint made of the number of event data + // properties and each of their EvtVariant type values. + // + // The first time we observe a new fingerprint value we get the XML + // representation of the event in order to know the parameter names. + // If they turn out to match the values that we got from the provider's + // metadata then we just associate the fingerprint with a pointer to the + // providers metadata for the event ID. + + defaultEM := s.Events[eventID] + + // Use XML to get the parameters names. + em, err := newEventMetadataFromEventHandle(s.Metadata, eventHandle) + if err != nil { + s.log.Debugw("Failed to make event metadata from event handle. Will "+ + "use default event metadata from the publisher.", + "event_id", eventID, + "fingerprint", eventDataFingerprint, + "error", err) + + if defaultEM != nil { + fingerprints[eventDataFingerprint] = defaultEM + } + return defaultEM + } + + // Are the parameters the same as what the provider metadata listed? + // (This ignores the message values.) + if em.equal(defaultEM) { + fingerprints[eventDataFingerprint] = defaultEM + return defaultEM + } + + // If we couldn't get a message from the event handle use the one + // from the installed provider metadata. + if defaultEM != nil && em.MsgStatic == "" && em.MsgTemplate == nil { + em.MsgStatic = defaultEM.MsgStatic + em.MsgTemplate = defaultEM.MsgTemplate + } + + s.log.Debugw("Obtained unique event metadata from event handle. "+ + "It differed from what was listed in the publisher's metadata.", + "event_id", eventID, + "fingerprint", eventDataFingerprint, + "default_event_metadata", defaultEM, + "event_metadata", em) + + fingerprints[eventDataFingerprint] = em + return em +} + +func (s *publisherMetadataStore) Close() error { + if s.Metadata != nil { + s.mutex.Lock() + defer s.mutex.Unlock() + + return s.Metadata.Close() + } + return nil +} + +type eventMetadata struct { + EventID uint16 // Event ID. + Version uint8 // Event format version. + MsgStatic string // Used when the message has no parameters. + MsgTemplate *template.Template `json:"-"` // Template that expects an array of values as its data. + EventData []eventData // Names of parameters from XML template. +} + +// newEventMetadataFromEventHandle collects metadata about an event type using +// the handle of an event. +func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle EvtHandle) (*eventMetadata, error) { + xml, err := getEventXML(publisher, eventHandle) + if err != nil { + return nil, err + } + + // By parsing the XML we can get the names of the parameters even if the + // publisher metadata is unavailable or is out of sync with the events. + event, err := sys.UnmarshalEventXML([]byte(xml)) + if err != nil { + return nil, errors.Wrap(err, "failed to unmarshal XML") + } + + em := &eventMetadata{ + EventID: uint16(event.EventIdentifier.ID), + Version: uint8(event.Version), + } + if len(event.EventData.Pairs) > 0 { + for _, pair := range event.EventData.Pairs { + em.EventData = append(em.EventData, eventData{Name: pair.Key}) + } + } else { + for _, pair := range event.UserData.Pairs { + em.EventData = append(em.EventData, eventData{Name: pair.Key}) + } + } + + // The message template is only available from the publisher metadata. This + // message template may not match up with the event data we got from the + // event's XML, but it's the only option available. Even forwarded events + // with "RenderedText" won't help because their messages are already + // rendered. + if publisher != nil { + msg, err := getMessageStringFromHandle(publisher, eventHandle, templateInserts.Slice()) + if err != nil { + return nil, err + } + if err = em.setMessage(msg); err != nil { + return nil, err + } + } + + return em, nil +} + +// newEventMetadataFromPublisherMetadata collects metadata about an event type +// using the publisher metadata. +func newEventMetadataFromPublisherMetadata(itr *EventMetadataIterator, publisher *PublisherMetadata) (*eventMetadata, error) { + em := &eventMetadata{} + err := multierr.Combine( + em.initEventID(itr), + em.initVersion(itr), + em.initEventDataTemplate(itr), + em.initEventMessage(itr, publisher), + ) + if err != nil { + return nil, err + } + return em, nil +} + +func (em *eventMetadata) initEventID(itr *EventMetadataIterator) error { + id, err := itr.EventID() + if err != nil { + return err + } + // The upper 16 bits are the qualifier and lower 16 are the ID. + em.EventID = uint16(0xFFFF & id) + return nil +} + +func (em *eventMetadata) initVersion(itr *EventMetadataIterator) error { + version, err := itr.Version() + if err != nil { + return err + } + em.Version = uint8(version) + return nil +} + +func (em *eventMetadata) initEventDataTemplate(itr *EventMetadataIterator) error { + xml, err := itr.Template() + if err != nil { + return err + } + // Some events do not have templates. + if xml == "" { + return nil + } + + tmpl := &eventTemplate{} + if err = tmpl.Unmarshal([]byte(xml)); err != nil { + return err + } + + for _, kv := range tmpl.Data { + kv.Name = eventDataNameTransform.Replace(kv.Name) + } + + em.EventData = tmpl.Data + return nil +} + +func (em *eventMetadata) initEventMessage(itr *EventMetadataIterator, publisher *PublisherMetadata) error { + messageID, err := itr.MessageID() + if err != nil { + return err + } + + msg, err := getMessageString(publisher, NilHandle, messageID, templateInserts.Slice()) + if err != nil { + return err + } + + return em.setMessage(msg) +} + +func (em *eventMetadata) setMessage(msg string) error { + msg = sys.RemoveWindowsLineEndings(msg) + tmplID := strconv.Itoa(int(em.EventID)) + + tmpl, err := template.New(tmplID).Funcs(eventMessageTemplateFuncs).Parse(msg) + if err != nil { + return err + } + + // One node means there were no parameters so this will optimize that case + // by using a static string rather than a text/template. + if len(tmpl.Root.Nodes) == 1 { + em.MsgStatic = msg + } else { + em.MsgTemplate = tmpl + } + return nil +} + +func (em *eventMetadata) equal(other *eventMetadata) bool { + if em == other { + return true + } + if em == nil || other == nil { + return false + } + + eventDataNamesEqual := func(a, b []eventData) bool { + if len(a) != len(b) { + return false + } + for n, v := range a { + if v.Name != b[n].Name { + return false + } + } + return true + } + + return em.EventID == other.EventID && + em.Version == other.Version && + eventDataNamesEqual(em.EventData, other.EventData) +} + +// --- Template Funcs + +// eventParam return an event data value inside a text/template. +func eventParam(items []interface{}, paramNumber int) (interface{}, error) { + // Windows parameter values start at %1 so adjust index value by -1. + index := paramNumber - 1 + if index < len(items) { + return items[index], nil + } + // Windows Event Viewer leaves the original placeholder (e.g. %22) in the + // rendered message when no value provided. + return "%" + strconv.Itoa(paramNumber), nil +} diff --git a/winlogbeat/sys/wineventlog/metadata_store_test.go b/winlogbeat/sys/wineventlog/metadata_store_test.go new file mode 100644 index 00000000000..0c89251bb7a --- /dev/null +++ b/winlogbeat/sys/wineventlog/metadata_store_test.go @@ -0,0 +1,63 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestPublisherMetadataStore(t *testing.T) { + logp.TestingSetup() + + s, err := newPublisherMetadataStore( + NilHandle, + "Microsoft-Windows-Security-Auditing", + logp.NewLogger("metadata")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + + assert.NotEmpty(t, s.Events) + assert.Empty(t, s.EventFingerprints) + + t.Run("event_metadata_from_handle", func(t *testing.T) { + log := openLog(t, security4752File) + defer log.Close() + + h := mustNextHandle(t, log) + defer h.Close() + + em, err := newEventMetadataFromEventHandle(s.Metadata, h) + if err != nil { + t.Fatal(err) + } + + assert.EqualValues(t, 4752, em.EventID) + assert.EqualValues(t, 0, em.Version) + assert.Empty(t, em.MsgStatic) + assert.NotNil(t, em.MsgTemplate) + assert.NotEmpty(t, em.EventData) + }) +} diff --git a/winlogbeat/sys/wineventlog/publisher_metadata.go b/winlogbeat/sys/wineventlog/publisher_metadata.go new file mode 100644 index 00000000000..73be91e849c --- /dev/null +++ b/winlogbeat/sys/wineventlog/publisher_metadata.go @@ -0,0 +1,663 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "os" + "syscall" + + "github.com/pkg/errors" + "go.uber.org/multierr" + "golang.org/x/sys/windows" +) + +// PublisherMetadata provides methods to query metadata from an event log +// publisher. +type PublisherMetadata struct { + Name string // Name of the publisher/provider. + Handle EvtHandle // Handle to the publisher metadata from EvtOpenPublisherMetadata. +} + +// Close releases the publisher metadata handle. +func (m *PublisherMetadata) Close() error { + return m.Handle.Close() +} + +// NewPublisherMetadata opens the publisher's metadata. Close must be called on +// the returned PublisherMetadata to release its handle. +func NewPublisherMetadata(session EvtHandle, name string) (*PublisherMetadata, error) { + var publisherName, logFile *uint16 + if info, err := os.Stat(name); err == nil && info.Mode().IsRegular() { + logFile, err = syscall.UTF16PtrFromString(name) + if err != nil { + return nil, err + } + } else { + publisherName, err = syscall.UTF16PtrFromString(name) + if err != nil { + return nil, err + } + } + + handle, err := _EvtOpenPublisherMetadata(session, publisherName, logFile, 0, 0) + if err != nil { + return nil, errors.Wrap(err, "failed in EvtOpenPublisherMetadata") + } + + return &PublisherMetadata{ + Name: name, + Handle: handle, + }, nil +} + +func (m *PublisherMetadata) stringProperty(propertyID EvtPublisherMetadataPropertyID) (string, error) { + v, err := EvtGetPublisherMetadataProperty(m.Handle, propertyID) + if err != nil { + return "", err + } + switch t := v.(type) { + case string: + return t, nil + case nil: + return "", nil + default: + return "", errors.Errorf("unexpected data type: %T", v) + } +} + +func (m *PublisherMetadata) PublisherGUID() (windows.GUID, error) { + v, err := EvtGetPublisherMetadataProperty(m.Handle, EvtPublisherMetadataPublisherGuid) + if err != nil { + return windows.GUID{}, err + } + switch t := v.(type) { + case windows.GUID: + return t, nil + case nil: + return windows.GUID{}, nil + default: + return windows.GUID{}, errors.Errorf("unexpected data type: %T", v) + } +} + +func (m *PublisherMetadata) ResourceFilePath() (string, error) { + return m.stringProperty(EvtPublisherMetadataResourceFilePath) +} + +func (m *PublisherMetadata) ParameterFilePath() (string, error) { + return m.stringProperty(EvtPublisherMetadataParameterFilePath) +} + +func (m *PublisherMetadata) MessageFilePath() (string, error) { + return m.stringProperty(EvtPublisherMetadataMessageFilePath) +} + +func (m *PublisherMetadata) HelpLink() (string, error) { + return m.stringProperty(EvtPublisherMetadataHelpLink) +} + +func (m *PublisherMetadata) PublisherMessageID() (uint32, error) { + v, err := EvtGetPublisherMetadataProperty(m.Handle, EvtPublisherMetadataPublisherMessageID) + if err != nil { + return 0, err + } + return v.(uint32), nil +} + +func (m *PublisherMetadata) PublisherMessage() (string, error) { + messageID, err := m.PublisherMessageID() + if err != nil { + return "", err + } + if int32(messageID) == -1 { + return "", nil + } + return getMessageStringFromMessageID(m, messageID, nil) +} + +func (m *PublisherMetadata) Keywords() ([]MetadataKeyword, error) { + return NewMetadataKeywords(m.Handle) +} + +func (m *PublisherMetadata) Opcodes() ([]MetadataOpcode, error) { + return NewMetadataOpcodes(m.Handle) +} + +func (m *PublisherMetadata) Levels() ([]MetadataLevel, error) { + return NewMetadataLevels(m.Handle) +} + +func (m *PublisherMetadata) Tasks() ([]MetadataTask, error) { + return NewMetadataTasks(m.Handle) +} + +func (m *PublisherMetadata) Channels() ([]MetadataChannel, error) { + return NewMetadataChannels(m.Handle) +} + +func (m *PublisherMetadata) EventMetadataIterator() (*EventMetadataIterator, error) { + return NewEventMetadataIterator(m) +} + +type MetadataKeyword struct { + Name string + Mask uint64 + Message string + MessageID uint32 +} + +func NewMetadataKeywords(publisherMetadataHandle EvtHandle) ([]MetadataKeyword, error) { + v, err := EvtGetPublisherMetadataProperty(publisherMetadataHandle, EvtPublisherMetadataKeywords) + if err != nil { + return nil, err + } + + arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) + if !ok { + return nil, errors.Errorf("unexpected handle type: %T", v) + } + defer arrayHandle.Close() + + arrayLen, err := EvtGetObjectArraySize(arrayHandle) + if err != nil { + return nil, errors.Wrap(err, "failed to get keyword array length") + } + + var values []MetadataKeyword + for i := uint32(0); i < arrayLen; i++ { + md, err := NewMetadataKeyword(publisherMetadataHandle, arrayHandle, i) + if err != nil { + return nil, errors.Wrapf(err, "failed to get keyword at array index %v", i) + } + + values = append(values, *md) + } + + return values, nil +} + +func NewMetadataKeyword(publisherMetadataHandle EvtHandle, arrayHandle EvtObjectArrayPropertyHandle, index uint32) (*MetadataKeyword, error) { + v, err := EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataKeywordMessageID, index) + if err != nil { + return nil, err + } + messageID := v.(uint32) + + // The value is -1 if the keyword did not specify a message attribute. + var message string + if int32(messageID) != -1 { + message, err = evtFormatMessage(publisherMetadataHandle, NilHandle, messageID, nil, EvtFormatMessageId) + if err != nil { + return nil, err + } + } + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataKeywordName, index) + if err != nil { + return nil, err + } + name := v.(string) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataKeywordValue, index) + if err != nil { + return nil, err + } + valueMask := v.(uint64) + + return &MetadataKeyword{ + Name: name, + Mask: valueMask, + MessageID: messageID, + Message: message, + }, nil +} + +type MetadataOpcode struct { + Name string + Mask uint32 + MessageID uint32 + Message string +} + +func NewMetadataOpcodes(publisherMetadataHandle EvtHandle) ([]MetadataOpcode, error) { + v, err := EvtGetPublisherMetadataProperty(publisherMetadataHandle, EvtPublisherMetadataOpcodes) + if err != nil { + return nil, err + } + + arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) + if !ok { + return nil, errors.Errorf("unexpected handle type: %T", v) + } + defer arrayHandle.Close() + + arrayLen, err := EvtGetObjectArraySize(arrayHandle) + if err != nil { + return nil, errors.Wrap(err, "failed to get opcode array length") + } + + var values []MetadataOpcode + for i := uint32(0); i < arrayLen; i++ { + md, err := NewMetadataOpcode(publisherMetadataHandle, arrayHandle, i) + if err != nil { + return nil, errors.Wrapf(err, "failed to get opcode at array index %v", i) + } + + values = append(values, *md) + } + + return values, nil +} + +func NewMetadataOpcode(publisherMetadataHandle EvtHandle, arrayHandle EvtObjectArrayPropertyHandle, index uint32) (*MetadataOpcode, error) { + v, err := EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataOpcodeMessageID, index) + if err != nil { + return nil, err + } + messageID := v.(uint32) + + // The value is -1 if the opcode did not specify a message attribute. + var message string + if int32(messageID) != -1 { + message, err = evtFormatMessage(publisherMetadataHandle, NilHandle, messageID, nil, EvtFormatMessageId) + if err != nil { + return nil, err + } + } + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataOpcodeName, index) + if err != nil { + return nil, err + } + name := v.(string) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataOpcodeValue, index) + if err != nil { + return nil, err + } + valueMask := v.(uint32) + + return &MetadataOpcode{ + Name: name, + Mask: valueMask, + MessageID: messageID, + Message: message, + }, nil +} + +type MetadataLevel struct { + Name string + Mask uint32 + MessageID uint32 + Message string +} + +func NewMetadataLevels(publisherMetadataHandle EvtHandle) ([]MetadataLevel, error) { + v, err := EvtGetPublisherMetadataProperty(publisherMetadataHandle, EvtPublisherMetadataLevels) + if err != nil { + return nil, err + } + + arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) + if !ok { + return nil, errors.Errorf("unexpected handle type: %T", v) + } + defer arrayHandle.Close() + + arrayLen, err := EvtGetObjectArraySize(arrayHandle) + if err != nil { + return nil, errors.Wrap(err, "failed to get level array length") + } + + var values []MetadataLevel + for i := uint32(0); i < arrayLen; i++ { + md, err := NewMetadataLevel(publisherMetadataHandle, arrayHandle, i) + if err != nil { + return nil, errors.Wrapf(err, "failed to get level at array index %v", i) + } + + values = append(values, *md) + } + + return values, nil +} + +func NewMetadataLevel(publisherMetadataHandle EvtHandle, arrayHandle EvtObjectArrayPropertyHandle, index uint32) (*MetadataLevel, error) { + v, err := EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataLevelMessageID, index) + if err != nil { + return nil, err + } + messageID := v.(uint32) + + // The value is -1 if the level did not specify a message attribute. + var message string + if int32(messageID) != -1 { + message, err = evtFormatMessage(publisherMetadataHandle, NilHandle, messageID, nil, EvtFormatMessageId) + if err != nil { + return nil, err + } + } + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataLevelName, index) + if err != nil { + return nil, err + } + name := v.(string) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataLevelValue, index) + if err != nil { + return nil, err + } + valueMask := v.(uint32) + + return &MetadataLevel{ + Name: name, + Mask: valueMask, + MessageID: messageID, + Message: message, + }, nil +} + +type MetadataTask struct { + Name string + Mask uint32 + MessageID uint32 + Message string + EventGUID windows.GUID +} + +func NewMetadataTasks(publisherMetadataHandle EvtHandle) ([]MetadataTask, error) { + v, err := EvtGetPublisherMetadataProperty(publisherMetadataHandle, EvtPublisherMetadataTasks) + if err != nil { + return nil, err + } + + arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) + if !ok { + return nil, errors.Errorf("unexpected handle type: %T", v) + } + defer arrayHandle.Close() + + arrayLen, err := EvtGetObjectArraySize(arrayHandle) + if err != nil { + return nil, errors.Wrap(err, "failed to get task array length") + } + + var values []MetadataTask + for i := uint32(0); i < arrayLen; i++ { + md, err := NewMetadataTask(publisherMetadataHandle, arrayHandle, i) + if err != nil { + return nil, errors.Wrapf(err, "failed to get task at array index %v", i) + } + + values = append(values, *md) + } + + return values, nil +} + +func NewMetadataTask(publisherMetadataHandle EvtHandle, arrayHandle EvtObjectArrayPropertyHandle, index uint32) (*MetadataTask, error) { + v, err := EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataTaskMessageID, index) + if err != nil { + return nil, err + } + messageID := v.(uint32) + + // The value is -1 if the task did not specify a message attribute. + var message string + if int32(messageID) != -1 { + message, err = evtFormatMessage(publisherMetadataHandle, NilHandle, messageID, nil, EvtFormatMessageId) + if err != nil { + return nil, err + } + } + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataTaskName, index) + if err != nil { + return nil, err + } + name := v.(string) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataTaskValue, index) + if err != nil { + return nil, err + } + valueMask := v.(uint32) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataTaskEventGuid, index) + if err != nil { + return nil, err + } + guid := v.(windows.GUID) + + return &MetadataTask{ + Name: name, + Mask: valueMask, + MessageID: messageID, + Message: message, + EventGUID: guid, + }, nil +} + +type MetadataChannel struct { + Name string + Index uint32 + ID uint32 + Message string + MessageID uint32 +} + +func NewMetadataChannels(publisherMetadataHandle EvtHandle) ([]MetadataChannel, error) { + v, err := EvtGetPublisherMetadataProperty(publisherMetadataHandle, EvtPublisherMetadataChannelReferences) + if err != nil { + return nil, err + } + + arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) + if !ok { + return nil, errors.Errorf("unexpected handle type: %T", v) + } + defer arrayHandle.Close() + + arrayLen, err := EvtGetObjectArraySize(arrayHandle) + if err != nil { + return nil, errors.Wrap(err, "failed to get task array length") + } + + var values []MetadataChannel + for i := uint32(0); i < arrayLen; i++ { + md, err := NewMetadataChannel(publisherMetadataHandle, arrayHandle, i) + if err != nil { + return nil, errors.Wrapf(err, "failed to get task at array index %v", i) + } + + values = append(values, *md) + } + + return values, nil +} + +func NewMetadataChannel(publisherMetadataHandle EvtHandle, arrayHandle EvtObjectArrayPropertyHandle, index uint32) (*MetadataChannel, error) { + v, err := EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataChannelReferenceMessageID, index) + if err != nil { + return nil, err + } + messageID := v.(uint32) + + // The value is -1 if the task did not specify a message attribute. + var message string + if int32(messageID) != -1 { + message, err = evtFormatMessage(publisherMetadataHandle, NilHandle, messageID, nil, EvtFormatMessageId) + if err != nil { + return nil, err + } + } + + // Channel name. + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataChannelReferencePath, index) + if err != nil { + return nil, err + } + name := v.(string) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataChannelReferenceIndex, index) + if err != nil { + return nil, err + } + channelIndex := v.(uint32) + + v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataChannelReferenceID, index) + if err != nil { + return nil, err + } + id := v.(uint32) + + return &MetadataChannel{ + Name: name, + Index: channelIndex, + ID: id, + MessageID: messageID, + Message: message, + }, nil +} + +type EventMetadataIterator struct { + Publisher *PublisherMetadata + eventMetadataEnumHandle EvtHandle + currentEvent EvtHandle + lastErr error +} + +func NewEventMetadataIterator(publisher *PublisherMetadata) (*EventMetadataIterator, error) { + eventMetadataEnumHandle, err := _EvtOpenEventMetadataEnum(publisher.Handle, 0) + if err != nil { + return nil, errors.Wrap(err, "failed to open event metadata enumerator with EvtOpenEventMetadataEnum") + } + + return &EventMetadataIterator{ + Publisher: publisher, + eventMetadataEnumHandle: eventMetadataEnumHandle, + }, nil +} + +func (itr *EventMetadataIterator) Close() error { + return multierr.Combine( + _EvtClose(itr.eventMetadataEnumHandle), + _EvtClose(itr.currentEvent), + ) +} + +// Next advances to the next event handle. It returns false when there are +// no more items or an error occurred. You should call Err() to check for an +// error. +func (itr *EventMetadataIterator) Next() bool { + // Close existing handle. + itr.currentEvent.Close() + + var err error + itr.currentEvent, err = _EvtNextEventMetadata(itr.eventMetadataEnumHandle, 0) + if err != nil { + if err != windows.ERROR_NO_MORE_ITEMS { + itr.lastErr = errors.Wrap(err, "failed advancing to next event metadata handle") + } + return false + } + return true +} + +// Err returns an error if Next() failed due to an error. +func (itr *EventMetadataIterator) Err() error { + return itr.lastErr +} + +func (itr *EventMetadataIterator) uint32Property(propertyID EvtEventMetadataPropertyID) (uint32, error) { + v, err := GetEventMetadataProperty(itr.currentEvent, propertyID) + if err != nil { + return 0, err + } + return v.(uint32), nil +} + +func (itr *EventMetadataIterator) uint64Property(propertyID EvtEventMetadataPropertyID) (uint64, error) { + v, err := GetEventMetadataProperty(itr.currentEvent, propertyID) + if err != nil { + return 0, err + } + return v.(uint64), nil +} + +func (itr *EventMetadataIterator) stringProperty(propertyID EvtEventMetadataPropertyID) (string, error) { + v, err := GetEventMetadataProperty(itr.currentEvent, propertyID) + if err != nil { + return "", err + } + return v.(string), nil +} + +func (itr *EventMetadataIterator) EventID() (uint32, error) { + return itr.uint32Property(EventMetadataEventID) +} + +func (itr *EventMetadataIterator) Version() (uint32, error) { + return itr.uint32Property(EventMetadataEventVersion) +} + +func (itr *EventMetadataIterator) Channel() (uint32, error) { + return itr.uint32Property(EventMetadataEventVersion) +} + +func (itr *EventMetadataIterator) Level() (uint32, error) { + return itr.uint32Property(EventMetadataEventLevel) +} + +func (itr *EventMetadataIterator) Opcode() (uint32, error) { + return itr.uint32Property(EventMetadataEventOpcode) +} + +func (itr *EventMetadataIterator) Task() (uint32, error) { + return itr.uint32Property(EventMetadataEventTask) +} + +func (itr *EventMetadataIterator) Keyword() (uint64, error) { + return itr.uint64Property(EventMetadataEventKeyword) +} + +func (itr *EventMetadataIterator) MessageID() (uint32, error) { + return itr.uint32Property(EventMetadataEventMessageID) +} + +func (itr *EventMetadataIterator) Template() (string, error) { + return itr.stringProperty(EventMetadataEventTemplate) +} + +// Message returns the raw event description without doing any substitutions +// (e.g. the message will contain %1, %2, etc. as parameter placeholders). +func (itr *EventMetadataIterator) Message() (string, error) { + messageID, err := itr.MessageID() + if err != nil { + return "", err + } + // If the event definition does not specify a message, the value is –1. + if int32(messageID) == -1 { + return "", nil + } + + return getMessageStringFromMessageID(itr.Publisher, messageID, nil) +} diff --git a/winlogbeat/sys/wineventlog/publisher_metadata_test.go b/winlogbeat/sys/wineventlog/publisher_metadata_test.go new file mode 100644 index 00000000000..1f5f5a2b85c --- /dev/null +++ b/winlogbeat/sys/wineventlog/publisher_metadata_test.go @@ -0,0 +1,230 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "golang.org/x/sys/windows" +) + +func TestPublisherMetadata(t *testing.T) { + // Modern Application + testPublisherMetadata(t, "Microsoft-Windows-PowerShell") + // Modern Application that uses UserData in XML + testPublisherMetadata(t, "Microsoft-Windows-Eventlog") + // Classic with messages (no event-data XML templates). + testPublisherMetadata(t, "Microsoft-Windows-Security-SPP") + // Classic without message metadata (no event-data XML templates). + testPublisherMetadata(t, "Windows Error Reporting") +} + +func testPublisherMetadata(t *testing.T, provider string) { + t.Run(provider, func(t *testing.T) { + md, err := NewPublisherMetadata(NilHandle, provider) + if err != nil { + t.Fatalf("%+v", err) + } + defer md.Close() + + t.Run("publisher_guid", func(t *testing.T) { + v, err := md.PublisherGUID() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("PublisherGUID: %v", v) + }) + + t.Run("resource_file_path", func(t *testing.T) { + v, err := md.ResourceFilePath() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("ResourceFilePath: %v", v) + }) + + t.Run("parameter_file_path", func(t *testing.T) { + v, err := md.ParameterFilePath() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("ParameterFilePath: %v", v) + }) + + t.Run("message_file_path", func(t *testing.T) { + v, err := md.MessageFilePath() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("MessageFilePath: %v", v) + }) + + t.Run("help_link", func(t *testing.T) { + v, err := md.HelpLink() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("HelpLink: %v", v) + }) + + t.Run("publisher_message_id", func(t *testing.T) { + v, err := md.PublisherMessageID() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("PublisherMessageID: %v", v) + }) + + t.Run("publisher_message", func(t *testing.T) { + v, err := md.PublisherMessage() + if err != nil { + t.Fatalf("%+v", err) + } + t.Logf("PublisherMessage: %v", v) + }) + + t.Run("keywords", func(t *testing.T) { + values, err := md.Keywords() + if err != nil { + t.Fatalf("%+v", err) + } + + if testing.Verbose() { + for _, value := range values { + t.Logf("%+v", value) + } + } + }) + + t.Run("opcodes", func(t *testing.T) { + values, err := md.Opcodes() + if err != nil { + t.Fatalf("%+v", err) + } + + if testing.Verbose() { + for _, value := range values { + t.Logf("%+v", value) + } + } + }) + + t.Run("levels", func(t *testing.T) { + values, err := md.Levels() + if err != nil { + t.Fatalf("%+v", err) + } + + if testing.Verbose() { + for _, value := range values { + t.Logf("%+v", value) + } + } + }) + + t.Run("tasks", func(t *testing.T) { + values, err := md.Tasks() + if err != nil { + t.Fatalf("%+v", err) + } + + if testing.Verbose() { + for _, value := range values { + t.Logf("%+v", value) + } + } + }) + + t.Run("channels", func(t *testing.T) { + values, err := md.Channels() + if err != nil { + t.Fatalf("%+v", err) + } + + if testing.Verbose() { + for _, value := range values { + t.Logf("%+v", value) + } + } + }) + + t.Run("event_metadata", func(t *testing.T) { + itr, err := md.EventMetadataIterator() + if err != nil { + t.Fatalf("%+v", err) + } + defer itr.Close() + + for itr.Next() { + eventID, err := itr.EventID() + assert.NoError(t, err) + t.Logf("eventID=%v (id=%v, qualifier=%v)", eventID, + 0xFFFF&eventID, // Lower 16 bits are the event ID. + (0xFFFF0000&eventID)>>16) // Upper 16 bits are the qualifier. + + version, err := itr.Version() + assert.NoError(t, err) + t.Logf("version=%v", version) + + channel, err := itr.Channel() + assert.NoError(t, err) + t.Logf("channel=%v", channel) + + level, err := itr.Level() + assert.NoError(t, err) + t.Logf("level=%v", level) + + opcode, err := itr.Opcode() + assert.NoError(t, err) + t.Logf("opcode=%v", opcode) + + task, err := itr.Task() + assert.NoError(t, err) + t.Logf("task=%v", task) + + keyword, err := itr.Keyword() + assert.NoError(t, err) + t.Logf("keyword=%v", keyword) + + messageID, err := itr.MessageID() + assert.NoError(t, err) + t.Logf("messageID=%v", messageID) + + template, err := itr.Template() + assert.NoError(t, err) + t.Logf("template=%v", template) + + message, err := itr.Message() + assert.NoError(t, err) + t.Logf("message=%v", message) + } + if err = itr.Err(); err != nil { + t.Fatalf("%+v", err) + } + }) + }) +} + +func TestNewPublisherMetadataUnknown(t *testing.T) { + _, err := NewPublisherMetadata(NilHandle, "Fake-Publisher") + assert.Equal(t, windows.ERROR_FILE_NOT_FOUND, errors.Cause(err)) +} diff --git a/winlogbeat/sys/wineventlog/query.go b/winlogbeat/sys/wineventlog/query.go index 76512747c44..8b822600425 100644 --- a/winlogbeat/sys/wineventlog/query.go +++ b/winlogbeat/sys/wineventlog/query.go @@ -49,8 +49,8 @@ var ( // Query that identifies the source of the events and one or more selectors or // suppressors. type Query struct { - // Name of the channel or the path to the log file that contains the events - // to query. + // Name of the channel or the URI path to the log file that contains the + // events to query. The path to files must be a URI like file://C:/log.evtx. Log string IgnoreOlder time.Duration // Ignore records older than this time period. @@ -209,7 +209,7 @@ func (qp *queryParams) providerSelect(q Query) error { return nil } - var selects []string + selects := make([]string, 0, len(q.Provider)) for _, p := range q.Provider { selects = append(selects, fmt.Sprintf("@Name='%s'", p)) } diff --git a/winlogbeat/sys/wineventlog/renderer.go b/winlogbeat/sys/wineventlog/renderer.go new file mode 100644 index 00000000000..4a6fcc2fef1 --- /dev/null +++ b/winlogbeat/sys/wineventlog/renderer.go @@ -0,0 +1,437 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "encoding/binary" + "fmt" + "strconv" + "sync" + "text/template" + "time" + "unsafe" + + "github.com/cespare/xxhash/v2" + "github.com/pkg/errors" + "go.uber.org/multierr" + "golang.org/x/sys/windows" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/winlogbeat/sys" +) + +const ( + // keywordClassic indicates the log was published with the "classic" event + // logging API. + // https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.eventing.reader.standardeventkeywords?view=netframework-4.8 + keywordClassic = 0x80000000000000 +) + +// Renderer is used for converting event log handles into complete events. +type Renderer struct { + // Cache of publisher metadata. Maps publisher names to stored metadata. + metadataCache map[string]*publisherMetadataStore + // Mutex to guard the metadataCache. The other members are immutable. + mutex sync.RWMutex + + session EvtHandle // Session handle if working with remote log. + systemContext EvtHandle // Render context for system values. + userContext EvtHandle // Render context for user values (event data). + log *logp.Logger +} + +// NewRenderer returns a new Renderer. +func NewRenderer(session EvtHandle, log *logp.Logger) (*Renderer, error) { + systemContext, err := _EvtCreateRenderContext(0, 0, EvtRenderContextSystem) + if err != nil { + return nil, errors.Wrap(err, "failed in EvtCreateRenderContext for system context") + } + + userContext, err := _EvtCreateRenderContext(0, 0, EvtRenderContextUser) + if err != nil { + return nil, errors.Wrap(err, "failed in EvtCreateRenderContext for user context") + } + + return &Renderer{ + metadataCache: map[string]*publisherMetadataStore{}, + session: session, + systemContext: systemContext, + userContext: userContext, + log: log.Named("renderer"), + }, nil +} + +// Close closes all handles held by the Renderer. +func (r *Renderer) Close() error { + r.mutex.Lock() + defer r.mutex.Unlock() + + errs := []error{r.systemContext.Close(), r.userContext.Close()} + for _, md := range r.metadataCache { + if err := md.Close(); err != nil { + errs = append(errs, err) + } + } + return multierr.Combine(errs...) +} + +// Render renders the event handle into an Event. +func (r *Renderer) Render(handle EvtHandle) (*sys.Event, error) { + event := &sys.Event{} + + if err := r.renderSystem(handle, event); err != nil { + return nil, errors.Wrap(err, "failed to render system properties") + } + + // From this point on it will return both the event and any errors. It's + // critical to not drop data. + var errs []error + + // This always returns a non-nil value (even on error). + md, err := r.getPublisherMetadata(event.Provider.Name) + if err != nil { + errs = append(errs, err) + } + + // Associate raw system properties to names (e.g. level=2 to Error). + enrichRawValuesWithNames(md, event) + + eventData, fingerprint, err := r.renderUser(handle, event) + if err != nil { + errs = append(errs, errors.Wrap(err, "failed to render event data")) + } + + // Load cached event metadata or try to bootstrap it from the event's XML. + eventMeta := md.getEventMetadata(uint16(event.EventIdentifier.ID), fingerprint, handle) + + // Associate key names with the event data values. + r.addEventData(eventMeta, eventData, event) + + if event.Message, err = r.formatMessage(md, eventMeta, handle, eventData, uint16(event.EventIdentifier.ID)); err != nil { + errs = append(errs, errors.Wrap(err, "failed to get the event message string")) + } + + if len(errs) > 0 { + return event, multierr.Combine(errs...) + } + return event, nil +} + +// getPublisherMetadata return a publisherMetadataStore for the provider. It +// never returns nil, but may return an error if it couldn't open a publisher. +func (r *Renderer) getPublisherMetadata(publisher string) (*publisherMetadataStore, error) { + var err error + + // NOTE: This code uses double-check locking to elevate to a write-lock + // when a cache value needs initialized. + r.mutex.RLock() + + // Lookup cached value. + md, found := r.metadataCache[publisher] + if !found { + // Elevate to write lock. + r.mutex.RUnlock() + r.mutex.Lock() + defer r.mutex.Unlock() + + // Double-check if the condition changed while upgrading the lock. + md, found = r.metadataCache[publisher] + if found { + return md, nil + } + + // Load metadata from the publisher. + md, err = newPublisherMetadataStore(r.session, publisher, r.log) + if err != nil { + // Return an empty store on error (can happen in cases where the + // log was forwarded and the provider doesn't exist on collector). + md = newEmptyPublisherMetadataStore(publisher, r.log) + err = errors.Wrapf(err, "failed to load publisher metadata for %v "+ + "(returning an empty metadata store)", publisher) + } + r.metadataCache[publisher] = md + } else { + r.mutex.RUnlock() + } + + return md, err +} + +// renderSystem writes all the system context properties into the event. +func (r *Renderer) renderSystem(handle EvtHandle, event *sys.Event) error { + bb, propertyCount, err := r.render(r.systemContext, handle) + if err != nil { + return errors.Wrap(err, "failed to get system values") + } + defer bb.free() + + for i := 0; i < int(propertyCount); i++ { + property := EvtSystemPropertyID(i) + offset := i * int(sizeofEvtVariant) + evtVar := (*EvtVariant)(unsafe.Pointer(&bb.buf[offset])) + + data, err := evtVar.Data(bb.buf) + if err != nil || data == nil { + continue + } + + switch property { + case EvtSystemProviderName: + event.Provider.Name = data.(string) + case EvtSystemProviderGuid: + event.Provider.GUID = data.(windows.GUID).String() + case EvtSystemEventID: + event.EventIdentifier.ID = uint32(data.(uint16)) + case EvtSystemQualifiers: + event.EventIdentifier.Qualifiers = data.(uint16) + case EvtSystemLevel: + event.LevelRaw = data.(uint8) + case EvtSystemTask: + event.TaskRaw = data.(uint16) + case EvtSystemOpcode: + event.OpcodeRaw = data.(uint8) + case EvtSystemKeywords: + event.KeywordsRaw = sys.HexInt64(data.(hexInt64)) + case EvtSystemTimeCreated: + event.TimeCreated.SystemTime = data.(time.Time) + case EvtSystemEventRecordId: + event.RecordID = data.(uint64) + case EvtSystemActivityID: + event.Correlation.ActivityID = data.(windows.GUID).String() + case EvtSystemRelatedActivityID: + event.Correlation.RelatedActivityID = data.(windows.GUID).String() + case EvtSystemProcessID: + event.Execution.ProcessID = data.(uint32) + case EvtSystemThreadID: + event.Execution.ThreadID = data.(uint32) + case EvtSystemChannel: + event.Channel = data.(string) + case EvtSystemComputer: + event.Computer = data.(string) + case EvtSystemUserID: + sid := data.(*windows.SID) + event.User.Identifier = sid.String() + var accountType uint32 + event.User.Name, event.User.Domain, accountType, _ = sid.LookupAccount("") + event.User.Type = sys.SIDType(accountType) + case EvtSystemVersion: + event.Version = sys.Version(data.(uint8)) + } + } + + return nil +} + +// renderUser returns the event/user data values. This does not provide the +// parameter names. It computes a fingerprint of the values types to help the +// caller match the correct names to the returned values. +func (r *Renderer) renderUser(handle EvtHandle, event *sys.Event) (values []interface{}, fingerprint uint64, err error) { + bb, propertyCount, err := r.render(r.userContext, handle) + if err != nil { + return nil, 0, errors.Wrap(err, "failed to get user values") + } + defer bb.free() + + if propertyCount == 0 { + return nil, 0, nil + } + + // Fingerprint the argument types to help ensure we match these values with + // the correct event data parameter names. + argumentHash := xxhash.New() + binary.Write(argumentHash, binary.LittleEndian, propertyCount) + + values = make([]interface{}, propertyCount) + for i := 0; i < propertyCount; i++ { + offset := i * int(sizeofEvtVariant) + evtVar := (*EvtVariant)(unsafe.Pointer(&bb.buf[offset])) + binary.Write(argumentHash, binary.LittleEndian, uint32(evtVar.Type)) + + values[i], err = evtVar.Data(bb.buf) + if err != nil { + r.log.Warnw("Failed to read event/user data value. Using nil.", + "provider", event.Provider.Name, + "event_id", event.EventIdentifier.ID, + "value_index", i, + "value_type", evtVar.Type.String(), + "error", err, + ) + } + } + + return values, argumentHash.Sum64(), nil +} + +// render uses EvtRender to event data. The caller must free() the returned when +// done accessing the bytes. +func (r *Renderer) render(context EvtHandle, eventHandle EvtHandle) (*byteBuffer, int, error) { + var bufferUsed, propertyCount uint32 + + err := _EvtRender(context, eventHandle, EvtRenderEventValues, 0, nil, &bufferUsed, &propertyCount) + if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { + return nil, 0, errors.Wrap(err, "failed in EvtRender") + } + + if propertyCount == 0 { + return nil, 0, nil + } + + bb := newByteBuffer() + bb.Reserve(int(bufferUsed)) + + err = _EvtRender(context, eventHandle, EvtRenderEventValues, uint32(len(bb.buf)), &bb.buf[0], &bufferUsed, &propertyCount) + if err != nil { + bb.free() + return nil, 0, errors.Wrap(err, "failed in EvtRender") + } + + return bb, int(propertyCount), nil +} + +// addEventData adds the event/user data values to the event. +func (r *Renderer) addEventData(evtMeta *eventMetadata, values []interface{}, event *sys.Event) { + if len(values) == 0 { + return + } + + if evtMeta == nil { + r.log.Warnw("Event metadata not found.", + "provider", event.Provider.Name, + "event_id", event.EventIdentifier.ID) + } else if len(values) != len(evtMeta.EventData) { + r.log.Warnw("The number of event data parameters doesn't match the number "+ + "of parameters in the template.", + "provider", event.Provider.Name, + "event_id", event.EventIdentifier.ID, + "event_parameter_count", len(values), + "template_parameter_count", len(evtMeta.EventData), + "template_version", evtMeta.Version, + "event_version", event.Version) + } + + // Fallback to paramN naming when the value does not exist in event data. + // This can happen for legacy providers without manifests. This can also + // happen if the installed provider manifest doesn't match the version that + // produced the event (forwarded events, reading from evtx, or software was + // updated). If software was updated it could also be that this cached + // template is now stale. + paramName := func(idx int) string { + if evtMeta != nil && idx < len(evtMeta.EventData) { + return evtMeta.EventData[idx].Name + } + return "param" + strconv.Itoa(idx) + } + + for i, v := range values { + var strVal string + switch t := v.(type) { + case string: + strVal = t + case *windows.SID: + strVal = t.String() + default: + strVal = fmt.Sprintf("%v", v) + } + + event.EventData.Pairs = append(event.EventData.Pairs, sys.KeyValue{ + Key: paramName(i), + Value: strVal, + }) + } + + return +} + +// formatMessage adds the message to the event. +func (r *Renderer) formatMessage(publisherMeta *publisherMetadataStore, + eventMeta *eventMetadata, eventHandle EvtHandle, values []interface{}, + eventID uint16) (string, error) { + + if eventMeta != nil { + if eventMeta.MsgStatic != "" { + return eventMeta.MsgStatic, nil + } else if eventMeta.MsgTemplate != nil { + return r.formatMessageFromTemplate(eventMeta.MsgTemplate, values) + } + } + + // Fallback to the trying EvtFormatMessage mechanism. + // This is the path for forwarded events in RenderedText mode where the + // local publisher metadata is not present. NOTE that if the local publisher + // metadata exists it will be preferred over the RenderedText. A config + // option might be desirable to control this behavior. + r.log.Debugf("Falling back to EvtFormatMessage for event ID %d.", eventID) + return getMessageString(publisherMeta.Metadata, eventHandle, 0, nil) +} + +// formatMessageFromTemplate creates the message by executing the stored Go +// text/template with the event/user data values. +func (r *Renderer) formatMessageFromTemplate(msgTmpl *template.Template, values []interface{}) (string, error) { + bb := newByteBuffer() + defer bb.free() + + if err := msgTmpl.Execute(bb, values); err != nil { + return "", errors.Wrapf(err, "failed to execute template with data=%#v template=%v", values, msgTmpl.Root.String()) + } + + return string(bb.Bytes()), nil +} + +// enrichRawValuesWithNames adds the names associated with the raw system +// property values. It enriches the event with keywords, opcode, level, and +// task. The search order is defined in the EvtFormatMessage documentation. +func enrichRawValuesWithNames(publisherMeta *publisherMetadataStore, event *sys.Event) { + // Keywords. Each bit in the value can represent a keyword. + rawKeyword := int64(event.KeywordsRaw) + isClassic := keywordClassic&rawKeyword > 0 + for mask, keyword := range winMeta.Keywords { + if rawKeyword&mask > 0 { + event.Keywords = append(event.Keywords, keyword) + rawKeyword -= mask + } + } + for mask, keyword := range publisherMeta.Keywords { + if rawKeyword&mask > 0 { + event.Keywords = append(event.Keywords, keyword) + rawKeyword -= mask + } + } + + // Opcode (search in winmeta first). + var found bool + if !isClassic { + event.Opcode, found = winMeta.Opcodes[event.OpcodeRaw] + if !found { + event.Opcode = publisherMeta.Opcodes[event.OpcodeRaw] + } + } + + // Level (search in winmeta first). + event.Level, found = winMeta.Levels[event.LevelRaw] + if !found { + event.Level = publisherMeta.Levels[event.LevelRaw] + } + + // Task (fall-back to winmeta if not found). + event.Task, found = publisherMeta.Tasks[event.TaskRaw] + if !found { + event.Task = winMeta.Tasks[event.TaskRaw] + } +} diff --git a/winlogbeat/sys/wineventlog/renderer_test.go b/winlogbeat/sys/wineventlog/renderer_test.go new file mode 100644 index 00000000000..3dd80343803 --- /dev/null +++ b/winlogbeat/sys/wineventlog/renderer_test.go @@ -0,0 +1,291 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "bytes" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "text/template" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/winlogbeat/sys" +) + +func TestRenderer(t *testing.T) { + logp.TestingSetup() + + t.Run(filepath.Base(sysmon9File), func(t *testing.T) { + log := openLog(t, sysmon9File) + defer log.Close() + + r, err := NewRenderer(NilHandle, logp.L()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + events := renderAllEvents(t, log, r, true) + assert.NotEmpty(t, events) + + if t.Failed() { + logAsJSON(t, events) + } + }) + + t.Run(filepath.Base(security4752File), func(t *testing.T) { + log := openLog(t, security4752File) + defer log.Close() + + r, err := NewRenderer(NilHandle, logp.L()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + events := renderAllEvents(t, log, r, false) + if !assert.Len(t, events, 1) { + return + } + e := events[0] + + assert.EqualValues(t, 4752, e.EventIdentifier.ID) + assert.Equal(t, "Microsoft-Windows-Security-Auditing", e.Provider.Name) + assertEqualIgnoreCase(t, "{54849625-5478-4994-a5ba-3e3b0328c30d}", e.Provider.GUID) + assert.Equal(t, "DC_TEST2k12.TEST.SAAS", e.Computer) + assert.Equal(t, "Security", e.Channel) + assert.EqualValues(t, 3707686, e.RecordID) + + assert.Equal(t, e.Keywords, []string{"Audit Success"}) + + assert.EqualValues(t, 0, e.OpcodeRaw) + assert.Equal(t, "Info", e.Opcode) + + assert.EqualValues(t, 0, e.LevelRaw) + assert.Equal(t, "Information", e.Level) + + assert.EqualValues(t, 13827, e.TaskRaw) + assert.Equal(t, "Distribution Group Management", e.Task) + + assert.EqualValues(t, 492, e.Execution.ProcessID) + assert.EqualValues(t, 1076, e.Execution.ThreadID) + assert.Len(t, e.EventData.Pairs, 10) + + assert.NotEmpty(t, e.Message) + + if t.Failed() { + logAsJSON(t, events) + } + }) + + t.Run(filepath.Base(winErrorReportingFile), func(t *testing.T) { + log := openLog(t, winErrorReportingFile) + defer log.Close() + + r, err := NewRenderer(NilHandle, logp.L()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + events := renderAllEvents(t, log, r, false) + if !assert.Len(t, events, 1) { + return + } + e := events[0] + + assert.EqualValues(t, 1001, e.EventIdentifier.ID) + assert.Equal(t, "Windows Error Reporting", e.Provider.Name) + assert.Empty(t, e.Provider.GUID) + assert.Equal(t, "vagrant", e.Computer) + assert.Equal(t, "Application", e.Channel) + assert.EqualValues(t, 420107, e.RecordID) + + assert.Equal(t, e.Keywords, []string{"Classic"}) + + assert.EqualValues(t, 0, e.OpcodeRaw) + assert.Equal(t, "", e.Opcode) + + assert.EqualValues(t, 4, e.LevelRaw) + assert.Equal(t, "Information", e.Level) + + assert.EqualValues(t, 0, e.TaskRaw) + assert.Equal(t, "None", e.Task) + + assert.EqualValues(t, 0, e.Execution.ProcessID) + assert.EqualValues(t, 0, e.Execution.ThreadID) + assert.Len(t, e.EventData.Pairs, 23) + + assert.NotEmpty(t, e.Message) + + if t.Failed() { + logAsJSON(t, events) + } + }) +} + +func TestTemplateFunc(t *testing.T) { + tmpl := template.Must(template.New(""). + Funcs(eventMessageTemplateFuncs). + Parse(`Hello {{ eventParam $ 1 }}! Foo {{ eventParam $ 2 }}.`)) + + buf := new(bytes.Buffer) + err := tmpl.Execute(buf, []interface{}{"world"}) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "Hello world! Foo %2.", buf.String()) +} + +// renderAllEvents reads all events and renders them. +func renderAllEvents(t *testing.T, log EvtHandle, renderer *Renderer, ignoreMissingMetadataError bool) []*sys.Event { + t.Helper() + + var events []*sys.Event + for { + h, done := nextHandle(t, log) + if done { + break + } + + func() { + defer h.Close() + + evt, err := renderer.Render(h) + if err != nil { + md := renderer.metadataCache[evt.Provider.Name] + if !ignoreMissingMetadataError || md.Metadata != nil { + t.Fatalf("Render failed: %+v", err) + } + } + + events = append(events, evt) + }() + } + + return events +} + +// setLogSize set the maximum number of bytes that an event log can hold. +func setLogSize(t testing.TB, provider string, sizeBytes int) { + output, err := exec.Command("wevtutil.exe", "sl", "/ms:"+strconv.Itoa(sizeBytes), provider).CombinedOutput() + if err != nil { + t.Fatal("failed to set log size", err, string(output)) + } +} + +func BenchmarkRenderer(b *testing.B) { + writer, teardown := createLog(b) + defer teardown() + + const totalEvents = 1000000 + msg := strings.Repeat("Hello world! ", 21) + for i := 0; i < totalEvents; i++ { + writer.Info(10, msg) + } + + setup := func() (*EventIterator, *Renderer) { + log := openLog(b, winlogbeatTestLogName) + + itr, err := NewEventIterator(WithSubscription(log), WithBatchSize(1024)) + if err != nil { + log.Close() + b.Fatal(err) + } + + r, err := NewRenderer(NilHandle, logp.NewLogger("bench")) + if err != nil { + log.Close() + itr.Close() + b.Fatal(err) + } + + return itr, r + } + + b.Run("single_thread", func(b *testing.B) { + itr, r := setup() + defer itr.Close() + defer r.Close() + + count := atomic.NewUint64(0) + start := time.Now() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + // Get next handle. + h, ok := itr.Next() + if !ok { + b.Fatal("Ran out of events before benchmark was done.", itr.Err()) + } + + // Render it. + _, err := r.Render(h) + if err != nil { + b.Fatal(err) + } + + count.Inc() + } + + elapsed := time.Since(start) + b.ReportMetric(float64(count.Load())/elapsed.Seconds(), "events/sec") + }) + + b.Run("parallel8", func(b *testing.B) { + itr, r := setup() + defer itr.Close() + defer r.Close() + + count := atomic.NewUint64(0) + start := time.Now() + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + // Get next handle. + h, ok := itr.Next() + if !ok { + b.Fatal("Ran out of events before benchmark was done.", itr.Err()) + } + + // Render it. + _, err := r.Render(h) + if err != nil { + b.Fatal(err) + } + count.Inc() + } + }) + + elapsed := time.Since(start) + b.ReportMetric(float64(count.Load())/elapsed.Seconds(), "events/sec") + b.ReportMetric(float64(runtime.GOMAXPROCS(0)), "gomaxprocs") + }) +} diff --git a/winlogbeat/sys/wineventlog/stringinserts.go b/winlogbeat/sys/wineventlog/stringinserts.go new file mode 100644 index 00000000000..347e478b9bd --- /dev/null +++ b/winlogbeat/sys/wineventlog/stringinserts.go @@ -0,0 +1,85 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "strconv" + "unsafe" + + "golang.org/x/sys/windows" +) + +const ( + // maxInsertStrings is the maximum number of parameters supported in a + // Windows event message. + maxInsertStrings = 99 +) + +// templateInserts contains EvtVariant values that can be used to substitute +// Go text/template expressions into a Windows event message. +var templateInserts = newTemplateStringInserts() + +// stringsInserts holds EvtVariant values with type EvtVarTypeString. +type stringInserts struct { + // insertStrings are slices holding the strings in the EvtVariant (this must + // keep a reference to these to prevent GC of the strings as there is + // an unsafe reference to them in the evtVariants). + insertStrings [maxInsertStrings][]uint16 + evtVariants [maxInsertStrings]EvtVariant +} + +// Pointer returns a pointer the EvtVariant array. +func (si *stringInserts) Slice() []EvtVariant { + return si.evtVariants[:] +} + +// clear clears the pointers (and unsafe pointers) so that the memory can be +// garbage collected. +func (si *stringInserts) clear() { + for i := 0; i < len(si.evtVariants); i++ { + si.evtVariants[i] = EvtVariant{} + si.insertStrings[i] = nil + } +} + +// newTemplateStringInserts returns a stringInserts where each value is a +// Go text/template expression that references an event data parameter. +func newTemplateStringInserts() *stringInserts { + si := &stringInserts{} + + for i := 0; i < len(si.evtVariants); i++ { + // Use i+1 to keep our inserts numbered the same as Window's inserts. + strSlice, err := windows.UTF16FromString(`{{eventParam $ ` + strconv.Itoa(i+1) + `}}`) + if err != nil { + // This will never happen. + panic(err) + } + + si.insertStrings[i] = strSlice + si.evtVariants[i] = EvtVariant{ + Value: uintptr(unsafe.Pointer(&strSlice[0])), + Count: uint32(len(strSlice)), + Type: EvtVarTypeString, + } + si.evtVariants[i].Type = EvtVarTypeString + } + + return si +} diff --git a/winlogbeat/eventlog/common_test.go b/winlogbeat/sys/wineventlog/stringinserts_test.go similarity index 50% rename from winlogbeat/eventlog/common_test.go rename to winlogbeat/sys/wineventlog/stringinserts_test.go index a7f3f8b2cf1..ffeb2a473d6 100644 --- a/winlogbeat/eventlog/common_test.go +++ b/winlogbeat/sys/wineventlog/stringinserts_test.go @@ -15,36 +15,32 @@ // specific language governing permissions and limitations // under the License. -package eventlog +// +build windows + +package wineventlog import ( "testing" + "unsafe" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/winlogbeat/checkpoint" + "github.com/stretchr/testify/assert" + "golang.org/x/sys/windows" ) -type factory func(*common.Config) (EventLog, error) -type teardown func() +func TestStringInserts(t *testing.T) { + assert.NotNil(t, templateInserts) -func fatalErr(t *testing.T, err error) { - if err != nil { - t.Fatal(err) - } -} + si := newTemplateStringInserts() + defer si.clear() -func newTestEventLog(t *testing.T, factory factory, options map[string]interface{}) EventLog { - config, err := common.NewConfigFrom(options) - fatalErr(t, err) - eventLog, err := factory(config) - fatalErr(t, err) - return eventLog -} + // "The value of n can be a number between 1 and 99." + // https://docs.microsoft.com/en-us/windows/win32/eventlog/message-text-files + assert.Contains(t, windows.UTF16ToString(si.insertStrings[0]), " 1}") + assert.Contains(t, windows.UTF16ToString(si.insertStrings[maxInsertStrings-1]), " 99}") -func setupEventLog(t *testing.T, factory factory, recordID uint64, options map[string]interface{}) (EventLog, teardown) { - eventLog := newTestEventLog(t, factory, options) - fatalErr(t, eventLog.Open(checkpoint.EventLogState{ - RecordNumber: recordID, - })) - return eventLog, func() { fatalErr(t, eventLog.Close()) } + for i, evtVariant := range si.evtVariants { + assert.EqualValues(t, uintptr(unsafe.Pointer(&si.insertStrings[i][0])), evtVariant.Value) + assert.Len(t, si.insertStrings[i], int(evtVariant.Count)) + assert.Equal(t, evtVariant.Type, EvtVarTypeString) + } } diff --git a/winlogbeat/sys/wineventlog/syscall_windows.go b/winlogbeat/sys/wineventlog/syscall_windows.go index ed7dfa67224..13beb04fd85 100644 --- a/winlogbeat/sys/wineventlog/syscall_windows.go +++ b/winlogbeat/sys/wineventlog/syscall_windows.go @@ -18,24 +18,31 @@ package wineventlog import ( + "fmt" "syscall" + "time" + "unsafe" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" ) // EvtHandle is a handle to the event log. type EvtHandle uintptr +func (h EvtHandle) Close() error { + return _EvtClose(h) +} + +const NilHandle EvtHandle = 0 + // Event log error codes. // https://msdn.microsoft.com/en-us/library/windows/desktop/ms681382(v=vs.85).aspx const ( - ERROR_INSUFFICIENT_BUFFER syscall.Errno = 122 - ERROR_NO_MORE_ITEMS syscall.Errno = 259 - ERROR_NONE_MAPPED syscall.Errno = 1332 - RPC_S_INVALID_BOUND syscall.Errno = 1734 - ERROR_INVALID_OPERATION syscall.Errno = 4317 - ERROR_EVT_MESSAGE_NOT_FOUND syscall.Errno = 15027 - ERROR_EVT_MESSAGE_ID_NOT_FOUND syscall.Errno = 15028 - ERROR_EVT_UNRESOLVED_VALUE_INSERT syscall.Errno = 15029 - ERROR_EVT_UNRESOLVED_PARAMETER_INSERT syscall.Errno = 15030 + ERROR_INSUFFICIENT_BUFFER syscall.Errno = 122 + ERROR_NO_MORE_ITEMS syscall.Errno = 259 + RPC_S_INVALID_BOUND syscall.Errno = 1734 + ERROR_INVALID_OPERATION syscall.Errno = 4317 ) // EvtSubscribeFlag defines the possible values that specify when to start subscribing to events. @@ -184,7 +191,7 @@ const ( var evtSystemMap = map[EvtSystemPropertyID]string{ EvtSystemProviderName: "Provider Name", - EvtSystemProviderGuid: "Provider GUID", + EvtSystemProviderGuid: "Provider PublisherGUID", EvtSystemEventID: "Event ID", EvtSystemQualifiers: "Qualifiers", EvtSystemLevel: "Level", @@ -299,11 +306,308 @@ const ( EvtSeekStrict EvtSeekFlag = 0x10000 ) -// Add -trace to enable debug prints around syscalls. -//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsyscall_windows.go syscall_windows.go +type EvtVariantType uint32 + +const ( + EvtVarTypeNull EvtVariantType = iota + EvtVarTypeString + EvtVarTypeAnsiString + EvtVarTypeSByte + EvtVarTypeByte + EvtVarTypeInt16 + EvtVarTypeUInt16 + EvtVarTypeInt32 + EvtVarTypeUInt32 + EvtVarTypeInt64 + EvtVarTypeUInt64 + EvtVarTypeSingle + EvtVarTypeDouble + EvtVarTypeBoolean + EvtVarTypeBinary + EvtVarTypeGuid + EvtVarTypeSizeT + EvtVarTypeFileTime + EvtVarTypeSysTime + EvtVarTypeSid + EvtVarTypeHexInt32 + EvtVarTypeHexInt64 + EvtVarTypeEvtHandle EvtVariantType = 32 + EvtVarTypeEvtXml EvtVariantType = 35 +) + +var evtVariantTypeNames = map[EvtVariantType]string{ + EvtVarTypeNull: "null", + EvtVarTypeString: "string", + EvtVarTypeAnsiString: "ansi_string", + EvtVarTypeSByte: "signed_byte", + EvtVarTypeByte: "unsigned byte", + EvtVarTypeInt16: "int16", + EvtVarTypeUInt16: "uint16", + EvtVarTypeInt32: "int32", + EvtVarTypeUInt32: "uint32", + EvtVarTypeInt64: "int64", + EvtVarTypeUInt64: "uint64", + EvtVarTypeSingle: "float32", + EvtVarTypeDouble: "float64", + EvtVarTypeBoolean: "boolean", + EvtVarTypeBinary: "binary", + EvtVarTypeGuid: "guid", + EvtVarTypeSizeT: "size_t", + EvtVarTypeFileTime: "filetime", + EvtVarTypeSysTime: "systemtime", + EvtVarTypeSid: "sid", + EvtVarTypeHexInt32: "hex_int32", + EvtVarTypeHexInt64: "hex_int64", + EvtVarTypeEvtHandle: "evt_handle", + EvtVarTypeEvtXml: "evt_xml", +} + +func (t EvtVariantType) Mask() EvtVariantType { + return t & EvtVariantTypeMask +} + +func (t EvtVariantType) IsArray() bool { + return t&EvtVariantTypeArray > 0 +} + +func (t EvtVariantType) String() string { + return evtVariantTypeNames[t.Mask()] +} + +const ( + EvtVariantTypeMask = 0x7f + EvtVariantTypeArray = 128 +) + +type EvtVariant struct { + Value uintptr + Count uint32 + Type EvtVariantType +} + +var sizeofEvtVariant = unsafe.Sizeof(EvtVariant{}) + +type hexInt32 int32 + +func (n hexInt32) String() string { + return fmt.Sprintf("%#x", uint32(n)) +} + +type hexInt64 int64 + +func (n hexInt64) String() string { + return fmt.Sprintf("%#x", uint64(n)) +} + +func (v EvtVariant) Data(buf []byte) (interface{}, error) { + typ := v.Type.Mask() + switch typ { + case EvtVarTypeNull: + return nil, nil + case EvtVarTypeString: + addr := unsafe.Pointer(&buf[0]) + offset := v.Value - uintptr(addr) + s, err := UTF16BytesToString(buf[offset:]) + return s, err + case EvtVarTypeSByte: + return int8(v.Value), nil + case EvtVarTypeByte: + return uint8(v.Value), nil + case EvtVarTypeInt16: + return int16(v.Value), nil + case EvtVarTypeInt32: + return int32(v.Value), nil + case EvtVarTypeHexInt32: + return hexInt32(v.Value), nil + case EvtVarTypeInt64: + return int64(v.Value), nil + case EvtVarTypeHexInt64: + return hexInt64(v.Value), nil + case EvtVarTypeUInt16: + return uint16(v.Value), nil + case EvtVarTypeUInt32: + return uint32(v.Value), nil + case EvtVarTypeUInt64: + return uint64(v.Value), nil + case EvtVarTypeSingle: + return float32(v.Value), nil + case EvtVarTypeDouble: + return float64(v.Value), nil + case EvtVarTypeBoolean: + if v.Value == 0 { + return false, nil + } + return true, nil + case EvtVarTypeGuid: + addr := unsafe.Pointer(&buf[0]) + offset := v.Value - uintptr(addr) + guid := (*windows.GUID)(unsafe.Pointer(&buf[offset])) + copy := *guid + return copy, nil + case EvtVarTypeFileTime: + ft := (*windows.Filetime)(unsafe.Pointer(&v.Value)) + return time.Unix(0, ft.Nanoseconds()).UTC(), nil + case EvtVarTypeSid: + addr := unsafe.Pointer(&buf[0]) + offset := v.Value - uintptr(addr) + sidPtr := (*windows.SID)(unsafe.Pointer(&buf[offset])) + return sidPtr.Copy() + case EvtVarTypeEvtHandle: + return EvtHandle(v.Value), nil + default: + return nil, errors.Errorf("unhandled type: %d", typ) + } +} + +type EvtEventMetadataPropertyID uint32 + +const ( + EventMetadataEventID EvtEventMetadataPropertyID = iota + EventMetadataEventVersion + EventMetadataEventChannel + EventMetadataEventLevel + EventMetadataEventOpcode + EventMetadataEventTask + EventMetadataEventKeyword + EventMetadataEventMessageID + EventMetadataEventTemplate +) + +type EvtPublisherMetadataPropertyID uint32 + +const ( + EvtPublisherMetadataPublisherGuid EvtPublisherMetadataPropertyID = iota + EvtPublisherMetadataResourceFilePath + EvtPublisherMetadataParameterFilePath + EvtPublisherMetadataMessageFilePath + EvtPublisherMetadataHelpLink + EvtPublisherMetadataPublisherMessageID + EvtPublisherMetadataChannelReferences + EvtPublisherMetadataChannelReferencePath + EvtPublisherMetadataChannelReferenceIndex + EvtPublisherMetadataChannelReferenceID + EvtPublisherMetadataChannelReferenceFlags + EvtPublisherMetadataChannelReferenceMessageID + EvtPublisherMetadataLevels + EvtPublisherMetadataLevelName + EvtPublisherMetadataLevelValue + EvtPublisherMetadataLevelMessageID + EvtPublisherMetadataTasks + EvtPublisherMetadataTaskName + EvtPublisherMetadataTaskEventGuid + EvtPublisherMetadataTaskValue + EvtPublisherMetadataTaskMessageID + EvtPublisherMetadataOpcodes + EvtPublisherMetadataOpcodeName + EvtPublisherMetadataOpcodeValue + EvtPublisherMetadataOpcodeMessageID + EvtPublisherMetadataKeywords + EvtPublisherMetadataKeywordName + EvtPublisherMetadataKeywordValue + EvtPublisherMetadataKeywordMessageID +) + +func EvtGetPublisherMetadataProperty(publisherMetadataHandle EvtHandle, propertyID EvtPublisherMetadataPropertyID) (interface{}, error) { + var bufferUsed uint32 + err := _EvtGetPublisherMetadataProperty(publisherMetadataHandle, propertyID, 0, 0, nil, &bufferUsed) + if err != windows.ERROR_INSUFFICIENT_BUFFER { + return "", errors.Errorf("expected ERROR_INSUFFICIENT_BUFFER but got %v", err) + } + + buf := make([]byte, bufferUsed) + pEvtVariant := (*EvtVariant)(unsafe.Pointer(&buf[0])) + err = _EvtGetPublisherMetadataProperty(publisherMetadataHandle, propertyID, 0, uint32(len(buf)), pEvtVariant, &bufferUsed) + if err != nil { + return nil, errors.Wrap(err, "failed in EvtGetPublisherMetadataProperty") + } + + v, err := pEvtVariant.Data(buf) + if err != nil { + return nil, err + } + + switch t := v.(type) { + case EvtHandle: + return EvtObjectArrayPropertyHandle(t), nil + default: + return v, nil + } +} + +func EvtGetObjectArrayProperty(arrayHandle EvtObjectArrayPropertyHandle, propertyID EvtPublisherMetadataPropertyID, index uint32) (interface{}, error) { + var bufferUsed uint32 + err := _EvtGetObjectArrayProperty(arrayHandle, propertyID, index, 0, 0, nil, &bufferUsed) + if err != windows.ERROR_INSUFFICIENT_BUFFER { + return nil, errors.Wrap(err, "failed in EvtGetObjectArrayProperty, expected ERROR_INSUFFICIENT_BUFFER") + } + + buf := make([]byte, bufferUsed) + pEvtVariant := (*EvtVariant)(unsafe.Pointer(&buf[0])) + err = _EvtGetObjectArrayProperty(arrayHandle, propertyID, index, 0, uint32(len(buf)), pEvtVariant, &bufferUsed) + if err != nil { + return nil, errors.Wrap(err, "failed in EvtGetObjectArrayProperty") + } + + value, err := pEvtVariant.Data(buf) + if err != nil { + return nil, errors.Wrap(err, "failed to read EVT_VARIANT value") + } + return value, nil +} + +type EvtObjectArrayPropertyHandle uint32 + +func (h EvtObjectArrayPropertyHandle) Close() error { + return _EvtClose(EvtHandle(h)) +} + +func EvtGetObjectArraySize(handle EvtObjectArrayPropertyHandle) (uint32, error) { + var arrayLen uint32 + if err := _EvtGetObjectArraySize(handle, &arrayLen); err != nil { + return 0, err + } + return arrayLen, nil +} + +func GetEventMetadataProperty(metadataHandle EvtHandle, propertyID EvtEventMetadataPropertyID) (interface{}, error) { + var bufferUsed uint32 + err := _EvtGetEventMetadataProperty(metadataHandle, 8, 0, 0, nil, &bufferUsed) + if err != windows.ERROR_INSUFFICIENT_BUFFER { + return nil, errors.Errorf("expected ERROR_INSUFFICIENT_BUFFER but got %v", err) + } + + buf := make([]byte, bufferUsed) + pEvtVariant := (*EvtVariant)(unsafe.Pointer(&buf[0])) + err = _EvtGetEventMetadataProperty(metadataHandle, propertyID, 0, uint32(len(buf)), pEvtVariant, &bufferUsed) + if err != nil { + return nil, errors.Wrap(err, "_EvtGetEventMetadataProperty") + } + + return pEvtVariant.Data(buf) +} + +// EvtClearLog removes all events from the specified channel and writes them to +// the target log file. +func EvtClearLog(session EvtHandle, channelPath string, targetFilePath string) error { + channel, err := windows.UTF16PtrFromString(channelPath) + if err != nil { + return err + } + + var target *uint16 + if targetFilePath != "" { + target, err = windows.UTF16PtrFromString(targetFilePath) + if err != nil { + return err + } + } + + return _EvtClearLog(session, channel, target, 0) +} // Windows API calls //sys _EvtOpenLog(session EvtHandle, path *uint16, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtOpenLog +//sys _EvtClearLog(session EvtHandle, channelPath *uint16, targetFilePath *uint16, flags uint32) (err error) = wevtapi.EvtClearLog //sys _EvtQuery(session EvtHandle, path *uint16, query *uint16, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtQuery //sys _EvtSubscribe(session EvtHandle, signalEvent uintptr, channelPath *uint16, query *uint16, bookmark EvtHandle, context uintptr, callback syscall.Handle, flags EvtSubscribeFlag) (handle EvtHandle, err error) = wevtapi.EvtSubscribe //sys _EvtCreateBookmark(bookmarkXML *uint16) (handle EvtHandle, err error) = wevtapi.EvtCreateBookmark @@ -317,5 +621,9 @@ const ( //sys _EvtNextChannelPath(channelEnum EvtHandle, channelPathBufferSize uint32, channelPathBuffer *uint16, channelPathBufferUsed *uint32) (err error) = wevtapi.EvtNextChannelPath //sys _EvtFormatMessage(publisherMetadata EvtHandle, event EvtHandle, messageID uint32, valueCount uint32, values uintptr, flags EvtFormatMessageFlag, bufferSize uint32, buffer *byte, bufferUsed *uint32) (err error) = wevtapi.EvtFormatMessage //sys _EvtOpenPublisherMetadata(session EvtHandle, publisherIdentity *uint16, logFilePath *uint16, locale uint32, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtOpenPublisherMetadata - -//sys _StringFromGUID2(rguid *syscall.GUID, pStr *uint16, strSize uint32) (err error) = ole32.StringFromGUID2 +//sys _EvtGetPublisherMetadataProperty(publisherMetadata EvtHandle, propertyID EvtPublisherMetadataPropertyID, flags uint32, bufferSize uint32, variant *EvtVariant, bufferUsed *uint32) (err error) = wevtapi.EvtGetPublisherMetadataProperty +//sys _EvtGetEventMetadataProperty(eventMetadata EvtHandle, propertyID EvtEventMetadataPropertyID, flags uint32, bufferSize uint32, variant *EvtVariant, bufferUsed *uint32) (err error) = wevtapi.EvtGetEventMetadataProperty +//sys _EvtOpenEventMetadataEnum(publisherMetadata EvtHandle, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtOpenEventMetadataEnum +//sys _EvtNextEventMetadata(enumerator EvtHandle, flags uint32) (handle EvtHandle, err error) = wevtapi.EvtNextEventMetadata +//sys _EvtGetObjectArrayProperty(objectArray EvtObjectArrayPropertyHandle, propertyID EvtPublisherMetadataPropertyID, arrayIndex uint32, flags uint32, bufferSize uint32, evtVariant *EvtVariant, bufferUsed *uint32) (err error) = wevtapi.EvtGetObjectArrayProperty +//sys _EvtGetObjectArraySize(objectArray EvtObjectArrayPropertyHandle, arraySize *uint32) (err error) = wevtapi.EvtGetObjectArraySize diff --git a/winlogbeat/sys/wineventlog/template.go b/winlogbeat/sys/wineventlog/template.go new file mode 100644 index 00000000000..e5b5c9b99ae --- /dev/null +++ b/winlogbeat/sys/wineventlog/template.go @@ -0,0 +1,35 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package wineventlog + +import ( + "encoding/xml" +) + +type eventTemplate struct { + Data []eventData `xml:"data"` +} + +type eventData struct { + Name string `xml:"name,attr"` + Type string `xml:"outType,attr"` +} + +func (t *eventTemplate) Unmarshal(xmlData []byte) error { + return xml.Unmarshal(xmlData, t) +} diff --git a/winlogbeat/sys/wineventlog/template_test.go b/winlogbeat/sys/wineventlog/template_test.go new file mode 100644 index 00000000000..94b23fb9d1d --- /dev/null +++ b/winlogbeat/sys/wineventlog/template_test.go @@ -0,0 +1,43 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package wineventlog + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEventTemplateUnmarshal(t *testing.T) { + const xmlTemplate = ` + +` + + et := &eventTemplate{} + assert.NoError(t, et.Unmarshal([]byte(xmlTemplate))) + assert.Len(t, et.Data, 8) +} diff --git a/winlogbeat/sys/wineventlog/testdata/application-windows-error-reporting.evtx b/winlogbeat/sys/wineventlog/testdata/application-windows-error-reporting.evtx new file mode 100644 index 0000000000000000000000000000000000000000..c37324d05becd6515c5cfffdea6082e489ee3330 GIT binary patch literal 69632 zcmeHQZE##w8Gbj}Y}2GoT1o{O!4{#m9kb0QO*W}Anr<7Yv>5sUiWx#aHfn{7BCB((gN!T`%^o1UXM1uw-nkQ;AM~k9ScRi zz4d1|c%k3+-lSV?WO8 z@)Mj}k*5!@z2=i7=jG}3^}J(--Gs33N1QhY#KY@hEX$PCfw1L%npdtwp4O7%@s?+Q z`Nomh(+@{#BN3Swd7|gKTD^<=f4u+hYkz;~(TYu2AsD`c+>$sjK7!i9=Rlk7mSF&U9HvI@Ul(u*@`gz3h99NPh$ z8I@ri4NE^l#&JB1^Q&YJuI|D4 zdCW;f0dE^0AB0|-qeu5wN{5ra8?yQkY7|#DFiBb%FaJ(du7I*bxX1*ZBUd}M`k>?< zM7;#o7= zL=GZ~eyr4r)~tz2tgNOS&ku6*0-Vso-1o-{*@$4w6(+B&MkE@Il*#*Wz^9GN`u$Pa z>a2b}j>>Av7*B;flvPCLz@Xj8!ciCVn9v4mafx*bmpkK?NE%hV5n%>!>pq-)Kh~I5FZ~;1 zvJF;boL&lQ=Emg)h&X7Fl&J&&wNXd$_T-7Zu>Da|mk9FdscQq_OBA~=ZHYu(a=e?H2LFMangyJFUH?C>>sP%QDDXGWSf}M+v7ch29gt1>qUb2;$j^=fRWo&SM#t z8xYf8tiA8B)bL5@M2*8b#d~FJ#6F*h!cUloG`#o8P^N0^1U$&cK$D;6t5M}{hQzJV znJ+4a;F%Oke;b~;I3)9Y+&{kocJlOM-8u*vX&kdY@harPwkwU6Nu;+aM@tVSPR{>Q zNpHtc#+pEg9=sJ>vDbiWJl}&5J=peR z+vl#-V~`;d1#3R-OZ~S7E)!C9;#6`q&%R2oOD z>*}!W$KGn}t;R7CRUKB^I)S|wS9+4T(u@5ToNvZn5;3uKGDa_6x8vH;Y)>zrEAqq+ z59R*uu@h%K^HhF&y4b*ef8}IguaZ68vKl$aJ};ucoxJFYsQ*g zCRc*_kMo%WX@f}93z68wYDX2EeFb_I@Vg5lu@RG)pkh| z6edKX^yUfDXAVT-3ytL_fmc{RUR-?DC9y}!pap;?4ZAjDk3gywKfV&1W+y5BdGEqcM|uo#05hn*CZxBa{Ug~BLDJnQuVzjL%>OQRZYf8 zDwzi0q8bI@BqJP+aD01u(+J1O)Km~cJ{3`HgyU2xcq1IgfH?A7XC6)nB3+Dd%%Z6< z(R2z}j+h}UedeHPBOHxzRBD*p!t^kKWa$YfW9Bk3)rO}51mPz&rbOMIccas{~UVa z#NjYNPn?UfZKuCYMdX5j&ak??TEEV9C+0d`@Jt7O?XN5oao${uqjD+ObNci!+GVmI zy=83huEom#dNgfPsHReA%&_5;#MKn`+R!*^cjwx1W-SPrcHGG-tsRXXHl}Zu*r(v( zLMM>x{!)vJ>I=06)ID7fdeM^3^%%Mc+Ig-`d(f_;Yr+!W>4+V4S<;_p>7X!%b~w8o zh7gV&r=yU-Zj-e0B{~NQqAwBJSf0;Nj#p5%T!^zL=zoLd+_`h)DtEtj#nSDknw~)d z{{hc^DB~)`6Lb~w9fKkeF0ytx{;>Syf*AZPMxey7BMKNS!iXwdMD!oZcnO__@N}zj z3epQm{d_2aZhxK+5Ay$3oVdrAaQ7?Kus@Sc19@if`a;<6f%5SO1f*w4)zKcnnBAR6=5)=Ko)?r>>h)5;GrS1QZlRI>!}@tKCJ z(cAedor_@fRVv_-M0>L^I(yUasFWT?9}usBUG*{f4dh}uAX*i9$Jr_>pn=UGbY{v~ z&>ct{uzuqws89Lz+XE+`{@&#Jt=Ro4X+JsgG5p+eKvc5os2T+%XRT!WRp_e0m5hnU zn=~Tp6R2X=;5`4y(p`gCH=~wdci%1)GCEAqnIWyxfSZPB2kmEi+PBAp$L{&XbD8@7 z3EywC*!BqOXHvLex_47rsX^I%pO-9>t|A*Xmw7S+%C6%rrchxnbR_! z&Yp|nx#CzqQ|WsJ2@^`+&%`e;I&=G({_W$WZ~gs;A>gE-ekPTyI7u}Mz)9B6Wc^Im z&&1wmb8I)qc5`f>_yOe9db!N8UES(u4w|-pChKQ@z-|{^|WE zfBQm)chr8SV)Kqcw6aQ8Jgpi9;AsY#NGrR~OqdSkm5=!8x#ijg50^eYLAyyhuWY<>)}zM0U&#Lq z)_7&(mF@clej4wMN|Wt@Dds1h)7c9^m@o~z@^d~;S~LFT5O7iuudI?4C#gmOILUZr z(Ba< zpPnTzHpE^jd3u8USyi&8M>PsaPbvIa@ojOgKkLiBJr-ZK=i;nA!t`fV$!ZVPD8L@Z zD;uwDyt47igKJZzY} zqe@mhtr`X3Y2zJ@cQoG7c*nwM!FWgG9gTN1-m&Ps<9~ddbp1_>LcmEuyrW81oTM5B z;3VT6jdwKO(RfGW9Sb8x;~kB6G~TfrGxwz3jLJRO_G5yuJ~umZJ@(QV-#US*#d_X$ zR_%JsWyNu@F%J^%O%!ofZO+8F4)XKcyX)K>W8H4LtwA>vQFacpEtu|#GmzQ%tna4r zS@U>D`hmXcq+l@HB&*!k_h+pPttbrIV#kPmn*WO4jtK zMgi%Wsy}PvX^hXB=g)fFx5ujUzgc-AlfPl|$|_myp&A9)!+2%mm5o=HoWN66nwRn&Rn$ z|7w^X-f^=p*+bJ=jRNqr@s7qj=5uG0`7rWh+41m)n-?m7L@D!U&1F4j&J3y3 zIpSw0IQKNxVLYw-@J#OjH#H<9r%_8xE`4p_wm8SN--GLarKOC13NmIx5BzlI7GVPU z!9LW>O~B5*us0coIxva#K-S&ZTa9fJ*XnT8i=!m=>Jjb?c*oa$oHW$(Ob9qBh<8-U zij!2M0GwpJqw$W$I~wnpi-Xgy*O=INJKj%)`hxRuuy;Hg(X{!qp4x}N#%ImNax(`_ zPbZUAJTz_otfgm(6-}3RUZ?celUqgoIqOarg{Umqvk{oMH>;G`g4StTn@QjG#|lJUyMD;uwDyt47i z=Fd76FTMPJa^sbaSGI8yg1+K3e^z=!CxPGT*U)6r8U^5K;~kB6G~Us8$HHjR{8`PPb*fC3@s0!Vzvb6<9EGFHA%{4) zy*T!g_%(wmj00OV;anH?yAZD1eTS8BTw)SO^cf|ONY-^W@Q%ayUSXK{Xyr-~;-zsP zC%yU7Ya!sIAl^|WD^6040&tS?j>bD0?`XWE@s5R&qWQC$KdX(;YTwc8w`TSoJ;#_z z1LG4r@%yiShwF*TN&OlOn+9Dtaw^A3c%A0GSQ#a zy;qnHwhtCWL?V8A9(egjx0gOWLH?{NS<|B$1*E4G{;VSsrybCe3}?JXdb{%*#Vi{( zbG)qVdjlIk``t8twm*`L*3aZU4*Pt^RUcpZCu1ULzX;r=?kbU4C`&*nUXO3hPq w&m${Raqxqz#%imr%K2;0;Pv76sQ>K>f+6d!Z?#Ed+A19)b@o|y=0i0X-^#A|> literal 0 HcmV?d00001 diff --git a/winlogbeat/sys/wineventlog/util_test.go b/winlogbeat/sys/wineventlog/util_test.go new file mode 100644 index 00000000000..5fd90f08003 --- /dev/null +++ b/winlogbeat/sys/wineventlog/util_test.go @@ -0,0 +1,153 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/andrewkroh/sys/windows/svc/eventlog" + "github.com/stretchr/testify/assert" + "golang.org/x/sys/windows" +) + +const ( + winlogbeatTestLogName = "WinEventLogTestGo" + + security4752File = "../../../x-pack/winlogbeat/module/security/test/testdata/4752.evtx" + sysmon9File = "../../../x-pack/winlogbeat/module/sysmon/test/testdata/sysmon-9.01.evtx" + winErrorReportingFile = "testdata/application-windows-error-reporting.evtx" +) + +// createLog creates a new event log and returns a handle for writing events +// to the log. +func createLog(t testing.TB) (log *eventlog.Log, tearDown func()) { + const name = winlogbeatTestLogName + const source = "wineventlog_test" + + existed, err := eventlog.InstallAsEventCreate(name, source, eventlog.Error|eventlog.Warning|eventlog.Info) + if err != nil { + t.Fatal(err) + } + + if existed { + EvtClearLog(NilHandle, name, "") + } + + log, err = eventlog.Open(source) + if err != nil { + eventlog.RemoveSource(name, source) + eventlog.RemoveProvider(name) + t.Fatal(err) + } + + setLogSize(t, winlogbeatTestLogName, 1024*1024*1024) // 1 GiB + + tearDown = func() { + log.Close() + EvtClearLog(NilHandle, name, "") + eventlog.RemoveSource(name, source) + eventlog.RemoveProvider(name) + } + + return log, tearDown +} + +// openLog opens an event log or .evtx file for reading. +func openLog(t testing.TB, log string, eventIDFilters ...string) EvtHandle { + var ( + err error + path = log + flags EvtQueryFlag = EvtQueryReverseDirection + ) + + if info, err := os.Stat(log); err == nil && info.Mode().IsRegular() { + flags |= EvtQueryFilePath + } else { + flags |= EvtQueryChannelPath + } + + var query string + if len(eventIDFilters) > 0 { + // Convert to URI. + abs, err := filepath.Abs(log) + if err != nil { + t.Fatal(err) + } + path = "file://" + filepath.ToSlash(abs) + + query, err = Query{Log: path, EventID: strings.Join(eventIDFilters, ",")}.Build() + if err != nil { + t.Fatal(err) + } + path = "" + } + + h, err := EvtQuery(NilHandle, path, query, flags) + if err != nil { + t.Fatal("Failed to open log", log, err) + } + return h +} + +// nextHandle reads one handle from the log. It returns done=true when there +// are no more items to read. +func nextHandle(t *testing.T, log EvtHandle) (handle EvtHandle, done bool) { + var numReturned uint32 + var handles [1]EvtHandle + + err := _EvtNext(log, 1, &handles[0], 0, 0, &numReturned) + if err != nil { + if err == windows.ERROR_NO_MORE_ITEMS { + return NilHandle, true + } + t.Fatal(err) + } + + return handles[0], false +} + +// mustNextHandle reads one handle from the log. +func mustNextHandle(t *testing.T, log EvtHandle) EvtHandle { + h, done := nextHandle(t, log) + if done { + t.Fatal("No more items to read.") + } + return h +} + +func logAsJSON(t testing.TB, object interface{}) { + data, err := json.MarshalIndent(object, "", " ") + if err != nil { + t.Fatal(err) + } + t.Log(string(data)) +} + +func assertEqualIgnoreCase(t *testing.T, expected, actual string) { + t.Helper() + assert.Equal(t, + strings.ToLower(expected), + strings.ToLower(actual), + ) +} diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows_test.go b/winlogbeat/sys/wineventlog/wineventlog_windows_test.go index f5411f446b9..9701cfd3679 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows_test.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows_test.go @@ -108,7 +108,3 @@ func TestChannels(t *testing.T) { } } } - -func TestExtension(t *testing.T) { - filepath.Ext("sysmon") -} diff --git a/winlogbeat/sys/wineventlog/winmeta.go b/winlogbeat/sys/wineventlog/winmeta.go new file mode 100644 index 00000000000..140b1c00066 --- /dev/null +++ b/winlogbeat/sys/wineventlog/winmeta.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package wineventlog + +// winMeta contains the static values that are a common across Windows. These +// values are from winmeta.xml inside the Windows SDK. +var winMeta = &publisherMetadataStore{ + Keywords: map[int64]string{ + 0: "AnyKeyword", + 0x1000000000000: "Response Time", + 0x4000000000000: "WDI Diag", + 0x8000000000000: "SQM", + 0x10000000000000: "Audit Failure", + 0x20000000000000: "Audit Success", + 0x40000000000000: "Correlation Hint", + 0x80000000000000: "Classic", + }, + Opcodes: map[uint8]string{ + 0: "Info", + 1: "Start", + 2: "Stop", + 3: "DCStart", + 4: "DCStop", + 5: "Extension", + 6: "Reply", + 7: "Resume", + 8: "Suspend", + 9: "Send", + }, + Levels: map[uint8]string{ + 0: "Information", // "Log Always", but Event Viewer shows Information. + 1: "Critical", + 2: "Error", + 3: "Warning", + 4: "Information", + 5: "Verbose", + }, + Tasks: map[uint16]string{ + 0: "None", + }, +} diff --git a/winlogbeat/sys/wineventlog/zsyscall_windows.go b/winlogbeat/sys/wineventlog/zsyscall_windows.go index 0e15ef6a3d6..2dbe865c1a3 100644 --- a/winlogbeat/sys/wineventlog/zsyscall_windows.go +++ b/winlogbeat/sys/wineventlog/zsyscall_windows.go @@ -55,23 +55,28 @@ func errnoErr(e syscall.Errno) error { var ( modwevtapi = windows.NewLazySystemDLL("wevtapi.dll") - modole32 = windows.NewLazySystemDLL("ole32.dll") - procEvtOpenLog = modwevtapi.NewProc("EvtOpenLog") - procEvtQuery = modwevtapi.NewProc("EvtQuery") - procEvtSubscribe = modwevtapi.NewProc("EvtSubscribe") - procEvtCreateBookmark = modwevtapi.NewProc("EvtCreateBookmark") - procEvtUpdateBookmark = modwevtapi.NewProc("EvtUpdateBookmark") - procEvtCreateRenderContext = modwevtapi.NewProc("EvtCreateRenderContext") - procEvtRender = modwevtapi.NewProc("EvtRender") - procEvtClose = modwevtapi.NewProc("EvtClose") - procEvtSeek = modwevtapi.NewProc("EvtSeek") - procEvtNext = modwevtapi.NewProc("EvtNext") - procEvtOpenChannelEnum = modwevtapi.NewProc("EvtOpenChannelEnum") - procEvtNextChannelPath = modwevtapi.NewProc("EvtNextChannelPath") - procEvtFormatMessage = modwevtapi.NewProc("EvtFormatMessage") - procEvtOpenPublisherMetadata = modwevtapi.NewProc("EvtOpenPublisherMetadata") - procStringFromGUID2 = modole32.NewProc("StringFromGUID2") + procEvtOpenLog = modwevtapi.NewProc("EvtOpenLog") + procEvtClearLog = modwevtapi.NewProc("EvtClearLog") + procEvtQuery = modwevtapi.NewProc("EvtQuery") + procEvtSubscribe = modwevtapi.NewProc("EvtSubscribe") + procEvtCreateBookmark = modwevtapi.NewProc("EvtCreateBookmark") + procEvtUpdateBookmark = modwevtapi.NewProc("EvtUpdateBookmark") + procEvtCreateRenderContext = modwevtapi.NewProc("EvtCreateRenderContext") + procEvtRender = modwevtapi.NewProc("EvtRender") + procEvtClose = modwevtapi.NewProc("EvtClose") + procEvtSeek = modwevtapi.NewProc("EvtSeek") + procEvtNext = modwevtapi.NewProc("EvtNext") + procEvtOpenChannelEnum = modwevtapi.NewProc("EvtOpenChannelEnum") + procEvtNextChannelPath = modwevtapi.NewProc("EvtNextChannelPath") + procEvtFormatMessage = modwevtapi.NewProc("EvtFormatMessage") + procEvtOpenPublisherMetadata = modwevtapi.NewProc("EvtOpenPublisherMetadata") + procEvtGetPublisherMetadataProperty = modwevtapi.NewProc("EvtGetPublisherMetadataProperty") + procEvtGetEventMetadataProperty = modwevtapi.NewProc("EvtGetEventMetadataProperty") + procEvtOpenEventMetadataEnum = modwevtapi.NewProc("EvtOpenEventMetadataEnum") + procEvtNextEventMetadata = modwevtapi.NewProc("EvtNextEventMetadata") + procEvtGetObjectArrayProperty = modwevtapi.NewProc("EvtGetObjectArrayProperty") + procEvtGetObjectArraySize = modwevtapi.NewProc("EvtGetObjectArraySize") ) func _EvtOpenLog(session EvtHandle, path *uint16, flags uint32) (handle EvtHandle, err error) { @@ -87,6 +92,18 @@ func _EvtOpenLog(session EvtHandle, path *uint16, flags uint32) (handle EvtHandl return } +func _EvtClearLog(session EvtHandle, channelPath *uint16, targetFilePath *uint16, flags uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procEvtClearLog.Addr(), 4, uintptr(session), uintptr(unsafe.Pointer(channelPath)), uintptr(unsafe.Pointer(targetFilePath)), uintptr(flags), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + func _EvtQuery(session EvtHandle, path *uint16, query *uint16, flags uint32) (handle EvtHandle, err error) { r0, _, e1 := syscall.Syscall6(procEvtQuery.Addr(), 4, uintptr(session), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(query)), uintptr(flags), 0, 0) handle = EvtHandle(r0) @@ -250,8 +267,70 @@ func _EvtOpenPublisherMetadata(session EvtHandle, publisherIdentity *uint16, log return } -func _StringFromGUID2(rguid *syscall.GUID, pStr *uint16, strSize uint32) (err error) { - r1, _, e1 := syscall.Syscall(procStringFromGUID2.Addr(), 3, uintptr(unsafe.Pointer(rguid)), uintptr(unsafe.Pointer(pStr)), uintptr(strSize)) +func _EvtGetPublisherMetadataProperty(publisherMetadata EvtHandle, propertyID EvtPublisherMetadataPropertyID, flags uint32, bufferSize uint32, variant *EvtVariant, bufferUsed *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procEvtGetPublisherMetadataProperty.Addr(), 6, uintptr(publisherMetadata), uintptr(propertyID), uintptr(flags), uintptr(bufferSize), uintptr(unsafe.Pointer(variant)), uintptr(unsafe.Pointer(bufferUsed))) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func _EvtGetEventMetadataProperty(eventMetadata EvtHandle, propertyID EvtEventMetadataPropertyID, flags uint32, bufferSize uint32, variant *EvtVariant, bufferUsed *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procEvtGetEventMetadataProperty.Addr(), 6, uintptr(eventMetadata), uintptr(propertyID), uintptr(flags), uintptr(bufferSize), uintptr(unsafe.Pointer(variant)), uintptr(unsafe.Pointer(bufferUsed))) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func _EvtOpenEventMetadataEnum(publisherMetadata EvtHandle, flags uint32) (handle EvtHandle, err error) { + r0, _, e1 := syscall.Syscall(procEvtOpenEventMetadataEnum.Addr(), 2, uintptr(publisherMetadata), uintptr(flags), 0) + handle = EvtHandle(r0) + if handle == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func _EvtNextEventMetadata(enumerator EvtHandle, flags uint32) (handle EvtHandle, err error) { + r0, _, e1 := syscall.Syscall(procEvtNextEventMetadata.Addr(), 2, uintptr(enumerator), uintptr(flags), 0) + handle = EvtHandle(r0) + if handle == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func _EvtGetObjectArrayProperty(objectArray EvtObjectArrayPropertyHandle, propertyID EvtPublisherMetadataPropertyID, arrayIndex uint32, flags uint32, bufferSize uint32, evtVariant *EvtVariant, bufferUsed *uint32) (err error) { + r1, _, e1 := syscall.Syscall9(procEvtGetObjectArrayProperty.Addr(), 7, uintptr(objectArray), uintptr(propertyID), uintptr(arrayIndex), uintptr(flags), uintptr(bufferSize), uintptr(unsafe.Pointer(evtVariant)), uintptr(unsafe.Pointer(bufferUsed)), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func _EvtGetObjectArraySize(objectArray EvtObjectArrayPropertyHandle, arraySize *uint32) (err error) { + r1, _, e1 := syscall.Syscall(procEvtGetObjectArraySize.Addr(), 2, uintptr(objectArray), uintptr(unsafe.Pointer(arraySize)), 0) if r1 == 0 { if e1 != 0 { err = errnoErr(e1)