diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 71de44dec74..520f21ce48d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -61,6 +61,19 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Updated to Go 1.7.4. {pull}3277[3277] *Metricbeat* + +- Add experimental filebeat metricset in the beats module. {pull}2297[2297] +- Add experimental libbeat metricset in the beats module. {pull}2339[2339] +- Add experimental docker module. Provided by Ingensi and @douaejeouit based on dockbeat. +- Add username and password config options to the MongoDB module. {pull}2889[2889] +- Add username and password config options to the PostgreSQL module. {pull}2889[2890] +- Add system core metricset for Windows. {pull}2883[2883] +- Add a sample Redis Kibana dashboard. {pull}2916[2916] +- Add support for MongoDB 3.4 and WiredTiger metrics. {pull}2999[2999] +- Add experimental kafka module with partition metricset. {pull}2969[2969] +- Add raw config option for mysql/status metricset. {pull}3001[3001] +- Add experimental dbstats metricset to MongoDB module. {pull}3228[3228] +- Use persistent, direct connections to the configured nodes for MongoDB module. {pull}3228[3228] - Kafka module broker matching enhancements. {pull}3129[3129] - Add a couchbase module with metricsets for node, cluster and bucket. {pull}3081[3081] - Export number of cores for cpu module. {pull}3192[3192] diff --git a/metricbeat/_meta/beat.full.yml b/metricbeat/_meta/beat.full.yml index 9cd9665e474..e8c7d5a0ae5 100644 --- a/metricbeat/_meta/beat.full.yml +++ b/metricbeat/_meta/beat.full.yml @@ -147,7 +147,7 @@ metricbeat.modules: #------------------------------- MongoDB Module ------------------------------ #- module: mongodb - #metricsets: ["status"] + #metricsets: ["dbstats", "status"] #enabled: true #period: 10s @@ -164,7 +164,6 @@ metricbeat.modules: # Password to use when connecting to MongoDB. Empty by default. #password: pass - #-------------------------------- MySQL Module ------------------------------- #- module: mysql #metricsets: ["status"] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d08ea5b7565..c3c61afdca4 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2636,6 +2636,102 @@ MongoDB metrics. +[float] +== dbstats Fields + +dbstats provides an overview of a particular mongo database. This document is most concerned with data volumes of a database. + + + +[float] +=== mongodb.dbstats.avg_obj_size.bytes + +type: long + +format: bytes + +[float] +=== mongodb.dbstats.collections + +type: integer + +[float] +=== mongodb.dbstats.data_size.bytes + +type: long + +format: bytes + +[float] +=== mongodb.dbstats.db + +type: keyword + +[float] +=== mongodb.dbstats.file_size.bytes + +type: long + +format: bytes + +[float] +=== mongodb.dbstats.index_size.bytes + +type: long + +format: bytes + +[float] +=== mongodb.dbstats.indexes + +type: long + +[float] +=== mongodb.dbstats.num_extents + +type: long + +[float] +=== mongodb.dbstats.objects + +type: long + +[float] +=== mongodb.dbstats.storage_size.bytes + +type: long + +format: bytes + +[float] +=== mongodb.dbstats.ns_size_mb.mb + +type: long + + +[float] +=== mongodb.dbstats.data_file_version.major + +type: long + +[float] +=== mongodb.dbstats.data_file_version.minor + +type: long + + +[float] +=== mongodb.dbstats.extent_free_list.num + +type: long + +[float] +=== mongodb.dbstats.extent_free_list.size.bytes + +type: long + +format: bytes + [float] == status Fields diff --git a/metricbeat/docs/modules/mongodb.asciidoc b/metricbeat/docs/modules/mongodb.asciidoc index ab647286ac3..8b9b5f8d90e 100644 --- a/metricbeat/docs/modules/mongodb.asciidoc +++ b/metricbeat/docs/modules/mongodb.asciidoc @@ -64,7 +64,7 @@ in <>. Here is an example configuration: ---- metricbeat.modules: #- module: mongodb - #metricsets: ["status"] + #metricsets: ["dbstats", "status"] #enabled: true #period: 10s @@ -80,7 +80,6 @@ metricbeat.modules: # Password to use when connecting to MongoDB. Empty by default. #password: pass - ---- [float] @@ -88,7 +87,11 @@ metricbeat.modules: The following metricsets are available: +* <> + * <> +include::mongodb/dbstats.asciidoc[] + include::mongodb/status.asciidoc[] diff --git a/metricbeat/docs/modules/mongodb/dbstats.asciidoc b/metricbeat/docs/modules/mongodb/dbstats.asciidoc new file mode 100644 index 00000000000..c713122064a --- /dev/null +++ b/metricbeat/docs/modules/mongodb/dbstats.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-mongodb-dbstats]] +include::../../../module/mongodb/dbstats/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/mongodb/dbstats/_meta/data.json[] +---- diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 8d0782555c1..3d88f2937d8 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -28,6 +28,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/kafka/consumergroup" _ "github.com/elastic/beats/metricbeat/module/kafka/partition" _ "github.com/elastic/beats/metricbeat/module/mongodb" + _ "github.com/elastic/beats/metricbeat/module/mongodb/dbstats" _ "github.com/elastic/beats/metricbeat/module/mongodb/status" _ "github.com/elastic/beats/metricbeat/module/mysql" _ "github.com/elastic/beats/metricbeat/module/mysql/status" diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 5c04e23e6ec..d655806d98a 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -147,7 +147,7 @@ metricbeat.modules: #------------------------------- MongoDB Module ------------------------------ #- module: mongodb - #metricsets: ["status"] + #metricsets: ["dbstats", "status"] #enabled: true #period: 10s @@ -164,7 +164,6 @@ metricbeat.modules: # Password to use when connecting to MongoDB. Empty by default. #password: pass - #-------------------------------- MySQL Module ------------------------------- #- module: mysql #metricsets: ["status"] diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index 7eabc2993a0..f3d85fbb414 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -1510,6 +1510,93 @@ }, "mongodb": { "properties": { + "dbstats": { + "properties": { + "avg_obj_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "collections": { + "type": "long" + }, + "data_file_version": { + "properties": { + "major": { + "type": "long" + }, + "minor": { + "type": "long" + } + } + }, + "data_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "db": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "extent_free_list": { + "properties": { + "num": { + "type": "long" + }, + "size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "file_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "index_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "indexes": { + "type": "long" + }, + "ns_size_mb": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "num_extents": { + "type": "long" + }, + "objects": { + "type": "long" + }, + "storage_size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, "status": { "properties": { "asserts": { diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index 01aab4c3775..3764b42bf87 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -1496,6 +1496,92 @@ }, "mongodb": { "properties": { + "dbstats": { + "properties": { + "avg_obj_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "collections": { + "type": "long" + }, + "data_file_version": { + "properties": { + "major": { + "type": "long" + }, + "minor": { + "type": "long" + } + } + }, + "data_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "db": { + "ignore_above": 1024, + "type": "keyword" + }, + "extent_free_list": { + "properties": { + "num": { + "type": "long" + }, + "size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "file_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "index_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "indexes": { + "type": "long" + }, + "ns_size_mb": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "num_extents": { + "type": "long" + }, + "objects": { + "type": "long" + }, + "storage_size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, "status": { "properties": { "asserts": { diff --git a/metricbeat/module/mongodb/_meta/config.yml b/metricbeat/module/mongodb/_meta/config.yml index 34bd73c6158..41ec65ca96b 100644 --- a/metricbeat/module/mongodb/_meta/config.yml +++ b/metricbeat/module/mongodb/_meta/config.yml @@ -1,5 +1,5 @@ #- module: mongodb - #metricsets: ["status"] + #metricsets: ["dbstats", "status"] #enabled: true #period: 10s @@ -15,4 +15,3 @@ # Password to use when connecting to MongoDB. Empty by default. #password: pass - diff --git a/metricbeat/module/mongodb/dbstats/_meta/data.json b/metricbeat/module/mongodb/dbstats/_meta/data.json new file mode 100644 index 00000000000..78cb66a8b0f --- /dev/null +++ b/metricbeat/module/mongodb/dbstats/_meta/data.json @@ -0,0 +1,27 @@ +{ + "@timestamp": "2016-05-23T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "host": "mongodb:27017", + "module": "mongodb", + "name": "dbstats", + "rtt": 115 + }, + "mongodb": { + "dbstats": { + "avg_obj_size": 59, + "collections": 1, + "data_size": 59, + "db": "admin", + "index_size": 32768, + "indexes": 2, + "num_extents": 0, + "objects": 1, + "storage_size": 16384 + } + }, + "type": "metricsets" +} \ No newline at end of file diff --git a/metricbeat/module/mongodb/dbstats/_meta/docs.asciidoc b/metricbeat/module/mongodb/dbstats/_meta/docs.asciidoc new file mode 100644 index 00000000000..514a741ccd9 --- /dev/null +++ b/metricbeat/module/mongodb/dbstats/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== mongodb dbstats MetricSet + +This is the dbstats metricset of the module mongodb. diff --git a/metricbeat/module/mongodb/dbstats/_meta/fields.yml b/metricbeat/module/mongodb/dbstats/_meta/fields.yml new file mode 100644 index 00000000000..83f03a5c772 --- /dev/null +++ b/metricbeat/module/mongodb/dbstats/_meta/fields.yml @@ -0,0 +1,62 @@ +- name: dbstats + type: group + description: > + dbstats provides an overview of a particular mongo database. This document + is most concerned with data volumes of a database. + fields: + - name: avg_obj_size.bytes + type: long + format: bytes + + - name: collections + type: integer + + - name: data_size.bytes + type: long + format: bytes + + - name: db + type: keyword + + - name: file_size.bytes + type: long + format: bytes + + - name: index_size.bytes + type: long + format: bytes + + - name: indexes + type: long + + - name: num_extents + type: long + + - name: objects + type: long + + - name: storage_size.bytes + type: long + format: bytes + + - name: ns_size_mb.mb + type: long + + - name: data_file_version + type: group + fields: + - name: major + type: long + + - name: minor + type: long + + - name: extent_free_list + type: group + fields: + - name: num + type: long + + - name: size.bytes + type: long + format: bytes diff --git a/metricbeat/module/mongodb/dbstats/data.go b/metricbeat/module/mongodb/dbstats/data.go new file mode 100644 index 00000000000..7e1d3e599d4 --- /dev/null +++ b/metricbeat/module/mongodb/dbstats/data.go @@ -0,0 +1,34 @@ +package dbstats + +import ( + s "github.com/elastic/beats/metricbeat/schema" + c "github.com/elastic/beats/metricbeat/schema/mapstriface" +) + +var schema = s.Schema{ + "db": c.Str("db"), + "collections": c.Int("collections"), + "objects": c.Int("objects"), + "avg_obj_size": c.Int("avgObjSize"), + "data_size": c.Int("dataSize"), + "storage_size": c.Int("storageSize"), + "num_extents": c.Int("numExtents"), + "indexes": c.Int("indexes"), + "index_size": c.Int("indexSize"), + // mmapv1 only + "ns_size_mb": c.Int("nsSizeMB", s.Optional), + // mmapv1 only + "file_size": c.Int("fileSize", s.Optional), + // mmapv1 only + "data_file_version": c.Dict("dataFileVersion", s.Schema{ + "major": c.Int("major"), + "minor": c.Int("minor"), + }, c.DictOptional), + // mmapv1 only + "extent_free_list": c.Dict("extentFreeList", s.Schema{ + "num": c.Int("num"), + "size": c.Int("size"), + }, c.DictOptional), +} + +var eventMapping = schema.Apply diff --git a/metricbeat/module/mongodb/dbstats/dbstats.go b/metricbeat/module/mongodb/dbstats/dbstats.go new file mode 100644 index 00000000000..f8b1877e7b0 --- /dev/null +++ b/metricbeat/module/mongodb/dbstats/dbstats.go @@ -0,0 +1,92 @@ +package dbstats + +import ( + "errors" + + "gopkg.in/mgo.v2" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/mongodb" +) + +var debugf = logp.MakeDebug("mongodb.dbstats") + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + if err := mb.Registry.AddMetricSet("mongodb", "dbstats", New, mongodb.ParseURL); err != nil { + panic(err) + } +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + mongoSession *mgo.Session +} + +// New creates a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + logp.Warn("EXPERIMENTAL: The %v %v metricset is experimental", base.Module().Name(), base.Name()) + + dialInfo, err := mgo.ParseURL(base.HostData().URI) + if err != nil { + return nil, err + } + dialInfo.Timeout = base.Module().Config().Timeout + + // instantiate direct connections to each of the configured Mongo hosts + mongoSession, err := mongodb.NewDirectSession(dialInfo) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + mongoSession: mongoSession, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch() ([]common.MapStr, error) { + // events is the list of events collected from each of the databases. + var events []common.MapStr + + // Get the list of databases names, which we'll use to call db.stats() on each + dbNames, err := m.mongoSession.DatabaseNames() + if err != nil { + logp.Err("Error retrieving database names from Mongo instance") + return events, err + } + + // for each database, call db.stats() and append to events + for _, dbName := range dbNames { + db := m.mongoSession.DB(dbName) + + result := common.MapStr{} + + err := db.Run("dbStats", &result) + if err != nil { + logp.Err("Failed to retrieve stats for db %s", dbName) + continue + } + events = append(events, eventMapping(result)) + } + + if len(events) == 0 { + err = errors.New("Failed to retrieve dbStats from any databases") + logp.Err(err.Error()) + return events, err + } + + return events, nil +} diff --git a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go new file mode 100644 index 00000000000..e53b9dffd27 --- /dev/null +++ b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go @@ -0,0 +1,67 @@ +// +build integration + +package dbstats + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/mongodb" + "github.com/stretchr/testify/assert" +) + +func TestFetch(t *testing.T) { + f := mbtest.NewEventsFetcher(t, getConfig()) + events, err := f.Fetch() + if !assert.NoError(t, err) { + t.FailNow() + } + + for _, event := range events { + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) + + // Check a few event Fields + db := event["db"].(string) + assert.NotEqual(t, db, "") + + collections := event["collections"].(int64) + assert.True(t, collections > 0) + + objects := event["objects"].(int64) + assert.True(t, objects > 0) + + avgObjSize := event["avg_obj_size"].(int64) + assert.True(t, avgObjSize > 0) + + dataSize := event["data_size"].(int64) + assert.True(t, dataSize > 0) + + storageSize := event["storage_size"].(int64) + assert.True(t, storageSize > 0) + + numExtents := event["num_extents"].(int64) + assert.True(t, numExtents >= 0) + + indexes := event["indexes"].(int64) + assert.True(t, indexes >= 0) + + indexSize := event["index_size"].(int64) + assert.True(t, indexSize > 0) + } +} + +func TestData(t *testing.T) { + f := mbtest.NewEventsFetcher(t, getConfig()) + err := mbtest.WriteEvents(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "mongodb", + "metricsets": []string{"dbstats"}, + "hosts": []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()}, + } +} diff --git a/metricbeat/module/mongodb/mongodb.go b/metricbeat/module/mongodb/mongodb.go index 6cf8332601c..c29cd76c282 100644 --- a/metricbeat/module/mongodb/mongodb.go +++ b/metricbeat/module/mongodb/mongodb.go @@ -5,6 +5,7 @@ import ( "net/url" "strings" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" @@ -18,6 +19,8 @@ func init() { } } +// NewModule creates a new mb.Module instance and validates that at least one host has been +// specified func NewModule(base mb.BaseModule) (mb.Module, error) { // Validate that at least one host has been specified. config := struct { @@ -30,6 +33,7 @@ func NewModule(base mb.BaseModule) (mb.Module, error) { return &base, nil } +// ParseURL parses valid MongoDB URL strings into an mb.HostData instance func ParseURL(module mb.Module, host string) (mb.HostData, error) { c := struct { Username string `config:"username"` @@ -61,3 +65,23 @@ func ParseURL(module mb.Module, host string) (mb.HostData, error) { return parse.NewHostDataFromURL(u), nil } + +// NewDirectSession estbalishes direct connections with a list of hosts. It uses the supplied +// dialInfo parameter as a template for establishing more direct connections +func NewDirectSession(dialInfo *mgo.DialInfo) (*mgo.Session, error) { + + // make a copy + nodeDialInfo := *dialInfo + nodeDialInfo.Direct = true + nodeDialInfo.FailFast = true + + logp.Info("Connecting to MongoDB node at %v", nodeDialInfo.Addrs) + + session, err := mgo.DialWithInfo(&nodeDialInfo) + if err != nil { + logp.Err("Error establishing direct connection to mongo node at %v. Error output: %s", nodeDialInfo.Addrs, err.Error()) + return nil, err + } + + return session, nil +} diff --git a/metricbeat/module/mongodb/status/status.go b/metricbeat/module/mongodb/status/status.go index 35335a6f6b3..04c38e83bdd 100644 --- a/metricbeat/module/mongodb/status/status.go +++ b/metricbeat/module/mongodb/status/status.go @@ -6,7 +6,6 @@ import ( "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/mongodb" - "github.com/pkg/errors" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" ) @@ -25,11 +24,18 @@ func init() { } } +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. type MetricSet struct { mb.BaseMetricSet - dialInfo *mgo.DialInfo + mongoSession *mgo.Session } +// New creates a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { dialInfo, err := mgo.ParseURL(base.HostData().URI) if err != nil { @@ -37,23 +43,25 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } dialInfo.Timeout = base.Module().Config().Timeout + // instantiate direct connections to Mongo host + mongoSession, err := mongodb.NewDirectSession(dialInfo) + if err != nil { + return nil, err + } + return &MetricSet{ BaseMetricSet: base, - dialInfo: dialInfo, + mongoSession: mongoSession, }, nil } +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. func (m *MetricSet) Fetch() (common.MapStr, error) { - session, err := mgo.DialWithInfo(m.dialInfo) - if err != nil { - return nil, err - } - defer session.Close() - - session.SetMode(mgo.Monotonic, true) result := map[string]interface{}{} - if err := session.DB("admin").Run(bson.D{{"serverStatus", 1}}, &result); err != nil { - return nil, errors.Wrap(err, "mongodb fetch failed") + if err := m.mongoSession.DB("admin").Run(bson.D{{"serverStatus", 1}}, &result); err != nil { + return nil, err } return eventMapping(result), nil diff --git a/metricbeat/module/mongodb/testing.go b/metricbeat/module/mongodb/testing.go index 96d9a0892e1..876e14cfa7a 100644 --- a/metricbeat/module/mongodb/testing.go +++ b/metricbeat/module/mongodb/testing.go @@ -16,7 +16,7 @@ func GetEnvHost() string { return host } -// GetMongodbEnvPort returns the port of the Mongodb server to use for testing. +// GetEnvPort returns the port of the Mongodb server to use for testing. // It reads the value from the MONGODB_PORT environment variable and returns // 27017 if it is not set. func GetEnvPort() string {