Mark removed fields as optional in MongoDB

Several fields are not reported when the storage engine is set to
wiredTiger, so marking them as optional. Fixes elastic#2990.

metricbeat/module/mongodb/status/data.go

@@ -11,7 +11,7 @@ var schema = s.Schema{
 		"ms": c.Int("uptimeMillis"),
 	},
 	"local_time":         c.Time("localTime"),
-	"write_backs_queued": c.Bool("writeBacksQueued"),
+	"write_backs_queued": c.Bool("writeBacksQueued", s.Optional),
 	"asserts": c.Dict("asserts", s.Schema{
 		"regular":   c.Int("regular"),
 		"warning":   c.Int("warning"),
@@ -31,7 +31,7 @@ var schema = s.Schema{
 			"ms": c.Int("last_ms"),
 		},
 		"last_finished": c.Time("last_finished"),
-	}),
+	}, c.DictOptional),
 	"connections": c.Dict("connections", s.Schema{
 		"current":       c.Int("current"),
 		"available":     c.Int("available"),
@@ -57,9 +57,9 @@ var schema = s.Schema{
 			"commits":               s.Object{"ms": c.Int("commits")},
 			"commits_in_write_lock": s.Object{"ms": c.Int("commitsInWriteLock")},
 		}),
-	}),
+	}, c.DictOptional),
 	"extra_info": c.Dict("extra_info", s.Schema{
-		"heap_usage":  s.Object{"bytes": c.Int("heap_usage_bytes")},
+		"heap_usage":  s.Object{"bytes": c.Int("heap_usage_bytes", s.Optional)},
 		"page_faults": c.Int("page_faults"),
 	}),
 	"network": c.Dict("network", s.Schema{

metricbeat/schema/mapstriface/mapstriface.go

@@ -65,15 +65,18 @@ import (
 )
 
 type ConvMap struct {
-	Key    string        // The key in the data map
-	Schema schema.Schema // The schema describing how to convert the sub-map
+	Key      string        // The key in the data map
+	Schema   schema.Schema // The schema describing how to convert the sub-map
+	Optional bool
 }
 
 // Map drills down in the data dictionary by using the key
 func (convMap ConvMap) Map(key string, event common.MapStr, data map[string]interface{}) {
 	subData, ok := data[convMap.Key].(map[string]interface{})
 	if !ok {
-		logp.Err("Error accessing sub-dictionary `%s`", convMap.Key)
+		if !convMap.Optional {
+			logp.Err("Error accessing sub-dictionary `%s`", convMap.Key)
+		}
 		return
 	}
 
@@ -82,14 +85,14 @@ func (convMap ConvMap) Map(key string, event common.MapStr, data map[string]inte
 	event[key] = subEvent
 }
 
-func Dict(key string, s schema.Schema) ConvMap {
-	return ConvMap{Key: key, Schema: s}
+func Dict(key string, s schema.Schema, opts ...DictSchemaOption) ConvMap {
+	return dictSetOptions(ConvMap{Key: key, Schema: s}, opts)
 }
 
 func toStrFromNum(key string, data map[string]interface{}) (interface{}, error) {
 	emptyIface, exists := data[key]
 	if !exists {
-		return false, fmt.Errorf("Key not found")
+		return false, fmt.Errorf("Key %s not found", key)
 	}
 	switch emptyIface.(type) {
 	case int, int32, int64, uint, uint32, uint64, float32, float64:
@@ -110,7 +113,7 @@ func toStr(key string, data map[string]interface{}) (interface{}, error) {
 	emptyIface, err := common.MapStr(data).GetValue(key)
 	if err != nil {
 		fmt.Println(err)
-		return "", fmt.Errorf("Key not found")
+		return "", fmt.Errorf("Key %s not found", key)
 	}
 	str, ok := emptyIface.(string)
 	if !ok {
@@ -127,7 +130,7 @@ func Str(key string, opts ...schema.SchemaOption) schema.Conv {
 func toBool(key string, data map[string]interface{}) (interface{}, error) {
 	emptyIface, exists := data[key]
 	if !exists {
-		return false, fmt.Errorf("Key not found")
+		return false, fmt.Errorf("Key %s not found", key)
 	}
 	boolean, ok := emptyIface.(bool)
 	if !ok {
@@ -144,7 +147,7 @@ func Bool(key string, opts ...schema.SchemaOption) schema.Conv {
 func toInteger(key string, data map[string]interface{}) (interface{}, error) {
 	emptyIface, exists := data[key]
 	if !exists {
-		return 0, fmt.Errorf("Key not found")
+		return 0, fmt.Errorf("Key %s not found", key)
 	}
 	switch emptyIface.(type) {
 	case int64:
@@ -178,7 +181,7 @@ func Int(key string, opts ...schema.SchemaOption) schema.Conv {
 func toTime(key string, data map[string]interface{}) (interface{}, error) {
 	emptyIface, exists := data[key]
 	if !exists {
-		return common.Time(time.Unix(0, 0)), fmt.Errorf("Key not found")
+		return common.Time(time.Unix(0, 0)), fmt.Errorf("Key %s not found", key)
 	}
 	ts, ok := emptyIface.(time.Time)
 	if !ok {
@@ -191,3 +194,22 @@ func toTime(key string, data map[string]interface{}) (interface{}, error) {
 func Time(key string, opts ...schema.SchemaOption) schema.Conv {
 	return schema.SetOptions(schema.Conv{Key: key, Func: toTime}, opts)
 }
+
+// SchemaOption is for adding optional parameters to the conversion
+// functions
+type DictSchemaOption func(c ConvMap) ConvMap
+
+// The optional flag suppresses the error message in case the key
+// doesn't exist or results in an error.
+func DictOptional(c ConvMap) ConvMap {
+	c.Optional = true
+	return c
+}
+
+// setOptions adds the optional flags to the Conv object
+func dictSetOptions(c ConvMap, opts []DictSchemaOption) ConvMap {
+	for _, opt := range opts {
+		c = opt(c)
+	}
+	return c
+}