From 3d0c3f650b08e52ab86a02cc710c4013cd60280c Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Mon, 14 Nov 2016 17:47:07 +0100 Subject: [PATCH] Mark removed fields as optional in MongoDB Several fields are not reported when the storage engine is set to wiredTiger, so marking them as optional. Fixes #2990. --- metricbeat/module/mongodb/status/data.go | 8 ++-- metricbeat/schema/mapstriface/mapstriface.go | 42 +++++++++++++++----- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/metricbeat/module/mongodb/status/data.go b/metricbeat/module/mongodb/status/data.go index b743c3e55ef..4c4c4490b80 100644 --- a/metricbeat/module/mongodb/status/data.go +++ b/metricbeat/module/mongodb/status/data.go @@ -11,7 +11,7 @@ var schema = s.Schema{ "ms": c.Int("uptimeMillis"), }, "local_time": c.Time("localTime"), - "write_backs_queued": c.Bool("writeBacksQueued"), + "write_backs_queued": c.Bool("writeBacksQueued", s.Optional), "asserts": c.Dict("asserts", s.Schema{ "regular": c.Int("regular"), "warning": c.Int("warning"), @@ -31,7 +31,7 @@ var schema = s.Schema{ "ms": c.Int("last_ms"), }, "last_finished": c.Time("last_finished"), - }), + }, c.DictOptional), "connections": c.Dict("connections", s.Schema{ "current": c.Int("current"), "available": c.Int("available"), @@ -57,9 +57,9 @@ var schema = s.Schema{ "commits": s.Object{"ms": c.Int("commits")}, "commits_in_write_lock": s.Object{"ms": c.Int("commitsInWriteLock")}, }), - }), + }, c.DictOptional), "extra_info": c.Dict("extra_info", s.Schema{ - "heap_usage": s.Object{"bytes": c.Int("heap_usage_bytes")}, + "heap_usage": s.Object{"bytes": c.Int("heap_usage_bytes", s.Optional)}, "page_faults": c.Int("page_faults"), }), "network": c.Dict("network", s.Schema{ diff --git a/metricbeat/schema/mapstriface/mapstriface.go b/metricbeat/schema/mapstriface/mapstriface.go index 99733eb1bed..8dd93ab06e9 100644 --- a/metricbeat/schema/mapstriface/mapstriface.go +++ b/metricbeat/schema/mapstriface/mapstriface.go @@ -65,15 +65,18 @@ import ( ) type ConvMap struct { - Key string // The key in the data map - Schema schema.Schema // The schema describing how to convert the sub-map + Key string // The key in the data map + Schema schema.Schema // The schema describing how to convert the sub-map + Optional bool } // Map drills down in the data dictionary by using the key func (convMap ConvMap) Map(key string, event common.MapStr, data map[string]interface{}) { subData, ok := data[convMap.Key].(map[string]interface{}) if !ok { - logp.Err("Error accessing sub-dictionary `%s`", convMap.Key) + if !convMap.Optional { + logp.Err("Error accessing sub-dictionary `%s`", convMap.Key) + } return } @@ -82,14 +85,14 @@ func (convMap ConvMap) Map(key string, event common.MapStr, data map[string]inte event[key] = subEvent } -func Dict(key string, s schema.Schema) ConvMap { - return ConvMap{Key: key, Schema: s} +func Dict(key string, s schema.Schema, opts ...DictSchemaOption) ConvMap { + return dictSetOptions(ConvMap{Key: key, Schema: s}, opts) } func toStrFromNum(key string, data map[string]interface{}) (interface{}, error) { emptyIface, exists := data[key] if !exists { - return false, fmt.Errorf("Key not found") + return false, fmt.Errorf("Key %s not found", key) } switch emptyIface.(type) { case int, int32, int64, uint, uint32, uint64, float32, float64: @@ -110,7 +113,7 @@ func toStr(key string, data map[string]interface{}) (interface{}, error) { emptyIface, err := common.MapStr(data).GetValue(key) if err != nil { fmt.Println(err) - return "", fmt.Errorf("Key not found") + return "", fmt.Errorf("Key %s not found", key) } str, ok := emptyIface.(string) if !ok { @@ -127,7 +130,7 @@ func Str(key string, opts ...schema.SchemaOption) schema.Conv { func toBool(key string, data map[string]interface{}) (interface{}, error) { emptyIface, exists := data[key] if !exists { - return false, fmt.Errorf("Key not found") + return false, fmt.Errorf("Key %s not found", key) } boolean, ok := emptyIface.(bool) if !ok { @@ -144,7 +147,7 @@ func Bool(key string, opts ...schema.SchemaOption) schema.Conv { func toInteger(key string, data map[string]interface{}) (interface{}, error) { emptyIface, exists := data[key] if !exists { - return 0, fmt.Errorf("Key not found") + return 0, fmt.Errorf("Key %s not found", key) } switch emptyIface.(type) { case int64: @@ -178,7 +181,7 @@ func Int(key string, opts ...schema.SchemaOption) schema.Conv { func toTime(key string, data map[string]interface{}) (interface{}, error) { emptyIface, exists := data[key] if !exists { - return common.Time(time.Unix(0, 0)), fmt.Errorf("Key not found") + return common.Time(time.Unix(0, 0)), fmt.Errorf("Key %s not found", key) } ts, ok := emptyIface.(time.Time) if !ok { @@ -191,3 +194,22 @@ func toTime(key string, data map[string]interface{}) (interface{}, error) { func Time(key string, opts ...schema.SchemaOption) schema.Conv { return schema.SetOptions(schema.Conv{Key: key, Func: toTime}, opts) } + +// SchemaOption is for adding optional parameters to the conversion +// functions +type DictSchemaOption func(c ConvMap) ConvMap + +// The optional flag suppresses the error message in case the key +// doesn't exist or results in an error. +func DictOptional(c ConvMap) ConvMap { + c.Optional = true + return c +} + +// setOptions adds the optional flags to the Conv object +func dictSetOptions(c ConvMap, opts []DictSchemaOption) ConvMap { + for _, opt := range opts { + c = opt(c) + } + return c +}