Merge pull request #1218 from andrewkroh/feature/wlb-query-filters

Winlogbeat - Select events by level, event_id, and provider

.travis.yml

@@ -18,7 +18,6 @@ env:
     - TARGETS="-C libbeat testsuite"
     - TARGETS="-C topbeat testsuite"
     - TARGETS="-C filebeat testsuite"
-    - TARGETS="-C winlogbeat testsuite"
     - TARGETS="-C packetbeat testsuite"
     - TARGETS="-C metricbeat testsuite"
     - TARGETS="-C libbeat crosscompile"
@@ -36,8 +35,6 @@ matrix:
       env: TARGETS="-C filebeat crosscompile"
     - os: osx
       env: TARGETS="-C libbeat crosscompile"
-    - os: osx
-      env: TARGETS="-C winlogbeat testsuite"
     - os: osx
       env: TARGETS="-C winlogbeat crosscompile"
     - os: osx
@@ -86,5 +83,4 @@ after_success:
   - test -f packetbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f packetbeat/build/coverage/full.cov
   - test -f topbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f topbeat/build/coverage/full.cov
   - test -f libbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f libbeat/build/coverage/full.cov
-  - test -f winlogbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f winlogbeat/build/coverage/full.cov
   - test -f metricbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f metricbeat/build/coverage/full.cov

CHANGELOG.asciidoc

@@ -65,6 +65,7 @@ https://github.com/elastic/beats/compare/v1.1.2...master[Check the HEAD diff]
 - Omit `fields` from Filebeat events when null {issue}899[899]
 
 *Winlogbeat*
+- Fix invalid `event_id` on Windows XP and Windows 2003 {pull}1153[1153]
 
 
 ==== Added
@@ -105,6 +106,8 @@ https://github.com/elastic/beats/compare/v1.1.2...master[Check the HEAD diff]
 - Add additional data to the events published by Winlogbeat. The new fields are `activity_id`,
 `event_data`, `keywords`, `opcode`, `process_id`, `provider_guid`, `related_activity_id`,
 `task`, `thread_id`, `user_data`. and `version`. {issue}1053[1053]
+- Add `event_id`, `level`, and `provider` configuration options for filtering events {pull}1218[1218]
+- Add `include_xml` configuration option for including the raw XML with the event {pull}1218[1218]
 
 ==== Deprecated
 

winlogbeat/beater/winlogbeat.go

@@ -33,23 +33,17 @@ func init() {
 // Debug logging functions for this package.
 var (
 	debugf    = logp.MakeDebug("winlogbeat")
-	detailf   = logp.MakeDebug("winlogbeat_detail")
 	memstatsf = logp.MakeDebug("memstats")
 )
 
 // Time the application was started.
 var startTime = time.Now().UTC()
 
-type log struct {
-	config.EventLogConfig
-	eventLog eventlog.EventLog
-}
-
 // Winlogbeat is used to conform to the beat interface
 type Winlogbeat struct {
 	beat       *beat.Beat             // Common beat information.
 	config     *config.Settings       // Configuration settings.
-	eventLogs  []log                  // List of all event logs being monitored.
+	eventLogs  []eventlog.EventLog    // List of all event logs being monitored.
 	done       chan struct{}          // Channel to initiate shutdown of main event loop.
 	client     publisher.Client       // Interface to publish event.
 	checkpoint *checkpoint.Checkpoint // Persists event log state to disk.
@@ -116,15 +110,27 @@ func (eb *Winlogbeat) Setup(b *beat.Beat) error {
 			err := http.Serve(sock, nil)
 			if err != nil {
 				logp.Warn("Unable to launch HTTP service for metrics. %v", err)
-				return
 			}
 		}()
 	}
 
+	// Create the event logs. This will validate the event log specific
+	// configuration.
+	eb.eventLogs = make([]eventlog.EventLog, 0, len(eb.config.Winlogbeat.EventLogs))
+	for _, config := range eb.config.Winlogbeat.EventLogs {
+		eventLog, err := eventlog.New(config)
+		if err != nil {
+			return fmt.Errorf("Failed to create new event log. %v", err)
+		}
+		debugf("Initialized EventLog[%s]", eventLog.Name())
+
+		eb.eventLogs = append(eb.eventLogs, eventLog)
+	}
+
 	return nil
 }
 
-// Run is used within the beats interface to execute the winlogbeat.
+// Run is used within the beats interface to execute the Winlogbeat workers.
 func (eb *Winlogbeat) Run(b *beat.Beat) error {
 	persistedState := eb.checkpoint.States()
 
@@ -133,40 +139,17 @@ func (eb *Winlogbeat) Run(b *beat.Beat) error {
 	publishedEvents.Add("failures", 0)
 	ignoredEvents.Add("total", 0)
 
-	// TODO: If no event_logs are specified in the configuration, use the
-	// Windows registry to discover the available event logs.
-	eb.eventLogs = make([]log, 0, len(eb.config.Winlogbeat.EventLogs))
-	for _, eventLogConfig := range eb.config.Winlogbeat.EventLogs {
-		debugf("Initializing EventLog[%s]", eventLogConfig.Name)
-
-		eventLog, err := eventlog.New(eventlog.Config{
-			Name:          eventLogConfig.Name,
-			API:           eventLogConfig.API,
-			EventMetadata: eventLogConfig.EventMetadata,
-		})
-		if err != nil {
-			return fmt.Errorf("Failed to create new event log for %s. %v",
-				eventLogConfig.Name, err)
-		}
-
-		// Initialize per event log metrics.
-		publishedEvents.Add(eventLogConfig.Name, 0)
-		ignoredEvents.Add(eventLogConfig.Name, 0)
-
-		eb.eventLogs = append(eb.eventLogs, log{
-			EventLogConfig: eventLogConfig,
-			eventLog:       eventLog,
-		})
-	}
-
 	var wg sync.WaitGroup
 	for _, log := range eb.eventLogs {
-		state, _ := persistedState[log.Name]
-		ignoreOlder, _ := config.IgnoreOlderDuration(log.IgnoreOlder)
+		state, _ := persistedState[log.Name()]
+
+		// Initialize per event log metrics.
+		publishedEvents.Add(log.Name(), 0)
+		ignoredEvents.Add(log.Name(), 0)
 
 		// Start a goroutine for each event log.
 		wg.Add(1)
-		go eb.processEventLog(&wg, log.eventLog, state, ignoreOlder)
+		go eb.processEventLog(&wg, log, state)
 	}
 
 	wg.Wait()
@@ -201,7 +184,6 @@ func (eb *Winlogbeat) processEventLog(
 	wg *sync.WaitGroup,
 	api eventlog.EventLog,
 	state checkpoint.EventLogState,
-	ignoreOlder time.Duration,
 ) {
 	defer wg.Done()
 
@@ -243,21 +225,8 @@ loop:
 			continue
 		}
 
-		// Filter events.
-		var events []common.MapStr
+		events := make([]common.MapStr, 0, len(records))
 		for _, lr := range records {
-			// TODO: Move filters close to source. Short circuit processing
-			// of event if it is going to be filtered.
-			// TODO: Add a severity filter.
-			// TODO: Check the global IgnoreOlder filter.
-			if ignoreOlder != 0 && time.Since(lr.TimeCreated.SystemTime) > ignoreOlder {
-				detailf("EventLog[%s] ignore_older filter dropping event: %+v",
-					api.Name(), lr)
-				ignoredEvents.Add("total", 1)
-				ignoredEvents.Add(api.Name(), 1)
-				continue
-			}
-
 			events = append(events, lr.ToMapStr())
 		}
 

winlogbeat/config/config.go

@@ -7,9 +7,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"time"
 
-	"github.com/elastic/beats/libbeat/common"
 	"github.com/joeshaw/multierror"
 )
 
@@ -30,7 +28,7 @@ type Validator interface {
 // Settings is the root of the Winlogbeat configuration data hierarchy.
 type Settings struct {
 	Winlogbeat WinlogbeatConfig       `config:"winlogbeat"`
-	All        map[string]interface{} `config:",inline"`
+	Raw        map[string]interface{} `config:",inline"`
 }
 
 // Validate validates the Settings data and returns an error describing
@@ -41,7 +39,7 @@ func (s Settings) Validate() error {
 
 	// Check for invalid top-level keys.
 	var errs multierror.Errors
-	for k := range s.All {
+	for k := range s.Raw {
 		k = strings.ToLower(k)
 		i := sort.SearchStrings(validKeys, k)
 		if i >= len(validKeys) || validKeys[i] != k {
@@ -60,32 +58,21 @@ func (s Settings) Validate() error {
 
 // WinlogbeatConfig contains all of Winlogbeat configuration data.
 type WinlogbeatConfig struct {
-	IgnoreOlder  string           `config:"ignore_older"`
-	EventLogs    []EventLogConfig `config:"event_logs"`
-	Metrics      MetricsConfig    `config:"metrics"`
-	RegistryFile string           `config:"registry_file"`
+	EventLogs    []map[string]interface{} `config:"event_logs"`
+	Metrics      MetricsConfig            `config:"metrics"`
+	RegistryFile string                   `config:"registry_file"`
 }
 
 // Validate validates the WinlogbeatConfig data and returns an error describing
 // all problems or nil if there are none.
 func (ebc WinlogbeatConfig) Validate() error {
 	var errs multierror.Errors
-	if _, err := IgnoreOlderDuration(ebc.IgnoreOlder); err != nil {
-		errs = append(errs, fmt.Errorf("Invalid top level ignore_older value "+
-			"'%s' (%v)", ebc.IgnoreOlder, err))
-	}
 
 	if len(ebc.EventLogs) == 0 {
 		errs = append(errs, fmt.Errorf("At least one event log must be "+
 			"configured as part of event_logs"))
 	}
 
-	for _, eventLog := range ebc.EventLogs {
-		if err := eventLog.Validate(); err != nil {
-			errs = append(errs, err)
-		}
-	}
-
 	if err := ebc.Metrics.Validate(); err != nil {
 		errs = append(errs, err)
 	}
@@ -129,51 +116,3 @@ func (mc MetricsConfig) Validate() error {
 
 	return nil
 }
-
-// EventLogConfig holds the configuration data that specifies which event logs
-// to monitor.
-type EventLogConfig struct {
-	common.EventMetadata `config:",inline"`
-	Name                 string
-	IgnoreOlder          string `config:"ignore_older"`
-	API                  string
-}
-
-// Validate validates the EventLogConfig data and returns an error describing
-// any problems or nil.
-func (elc EventLogConfig) Validate() error {
-	var errs multierror.Errors
-	if elc.Name == "" {
-		err := fmt.Errorf("event log is missing a 'name'")
-		errs = append(errs, err)
-	}
-
-	if _, err := IgnoreOlderDuration(elc.IgnoreOlder); err != nil {
-		err := fmt.Errorf("Invalid ignore_older value ('%s') for event_log "+
-			"'%s' (%v)", elc.IgnoreOlder, elc.Name, err)
-		errs = append(errs, err)
-	}
-
-	switch strings.ToLower(elc.API) {
-	case "", "eventlogging", "wineventlog":
-		break
-	default:
-		err := fmt.Errorf("Invalid api value ('%s') for event_log '%s'",
-			elc.API, elc.Name)
-		errs = append(errs, err)
-	}
-
-	return errs.Err()
-}
-
-// IgnoreOlderDuration returns the parsed value of the IgnoreOlder string. If
-// IgnoreOlder is not set then (0, nil) is returned. If IgnoreOlder is not
-// parsable as a duration then an error is returned. See time.ParseDuration.
-func IgnoreOlderDuration(ignoreOlder string) (time.Duration, error) {
-	if ignoreOlder == "" {
-		return time.Duration(0), nil
-	}
-
-	duration, err := time.ParseDuration(ignoreOlder)
-	return duration, err
-}

winlogbeat/config/config_test.go

@@ -17,7 +17,10 @@ func (v validationTestCase) run(t *testing.T) {
 	if v.errMsg == "" {
 		assert.NoError(t, v.config.Validate())
 	} else {
-		assert.Contains(t, v.config.Validate().Error(), v.errMsg)
+		err := v.config.Validate()
+		if assert.Error(t, err, "expected '%s'", v.errMsg) {
+			assert.Contains(t, err.Error(), v.errMsg)
+		}
 	}
 }
 
@@ -26,17 +29,17 @@ func TestConfigValidate(t *testing.T) {
 		// Top-level config
 		{
 			WinlogbeatConfig{
-				EventLogs: []EventLogConfig{
-					{Name: "App"},
+				EventLogs: []map[string]interface{}{
+					{"Name": "App"},
 				},
 			},
 			"", // No Error
 		},
 		{
 			Settings{
 				WinlogbeatConfig{
-					EventLogs: []EventLogConfig{
-						{Name: "App"},
+					EventLogs: []map[string]interface{}{
+						{"Name": "App"},
 					},
 				},
 				map[string]interface{}{"other": "value"},
@@ -49,29 +52,15 @@ func TestConfigValidate(t *testing.T) {
 			"1 error: At least one event log must be configured as part of " +
 				"event_logs",
 		},
-		{
-			WinlogbeatConfig{IgnoreOlder: "1"},
-			"2 errors: Invalid top level ignore_older value '1' (time: " +
-				"missing unit in duration 1); At least one event log must be " +
-				"configured as part of event_logs",
-		},
 		{
 			WinlogbeatConfig{
-				EventLogs: []EventLogConfig{
-					{Name: "App"},
+				EventLogs: []map[string]interface{}{
+					{"Name": "App"},
 				},
 				Metrics: MetricsConfig{BindAddress: "example.com"},
 			},
 			"1 error: bind_address",
 		},
-		{
-			WinlogbeatConfig{
-				EventLogs: []EventLogConfig{
-					{},
-				},
-			},
-			"1 error: event log is missing a 'name'",
-		},
 		// MetricsConfig
 		{
 			MetricsConfig{},
@@ -103,32 +92,6 @@ func TestConfigValidate(t *testing.T) {
 			MetricsConfig{BindAddress: "example.com:65536"},
 			"bind_address port must be within [1-65535] but was '65536'",
 		},
-		// EventLogConfig
-		{
-			EventLogConfig{Name: "System"},
-			"",
-		},
-		{
-			EventLogConfig{},
-			"event log is missing a 'name'",
-		},
-		{
-			EventLogConfig{Name: "System", IgnoreOlder: "24"},
-			"Invalid ignore_older value ('24') for event_log 'System' " +
-				"(time: missing unit in duration 24)",
-		},
-		{
-			EventLogConfig{Name: "System", API: "eventlogging"},
-			"",
-		},
-		{
-			EventLogConfig{Name: "System", API: "wineventlog"},
-			"",
-		},
-		{
-			EventLogConfig{Name: "System", API: "invalid"},
-			"Invalid api value ('invalid') for event_log 'System'",
-		},
 	}
 
 	for _, test := range testCases {

winlogbeat/docs/configuring-howto.asciidoc

@@ -2,7 +2,8 @@
 == Configuring Winlogbeat
 
 After following the <<winlogbeat-configuration,configuration steps>> in the
-Getting Started, you might want to fine tune the behavior of Winlogbeat. This section
-describes some common use cases for changing configuration options. 
+Getting Started, you might want to fine tune the behavior of Winlogbeat. This
+section describes some common use cases for changing configuration options. 
 
-For a complete description of all Winlogbeat configuration options, see <<winlogbeat-configuration-details>>.
+For a complete description of all Winlogbeat configuration options, see
+<<winlogbeat-configuration-details>>.