Move elasticsearch node_stats metricset under node.stats namespace (#…

…4142)

So far the node_stats was putting all its data under `elasticsearch.node_stats.*`. This was changed to `elasticsearch.node.stats.*`.

Further changes:

* Remove support for setting index in metricset. This was never used and can now be done through using format string in the index setting.
* Remove support for setting type in metricset. Type will be removed in elasticsearch.
* Rename ModuleData constant to ModuleDataKey.
* Introduce NamespaceKey as constant to replace hardcoded `_namespace`.
* Cleanup event generation for metricbeat.
* Remove `type` from event as not needed and remove it as required field.

CHANGELOG.asciidoc

@@ -171,6 +171,7 @@ https://github.com/elastic/beats/compare/v5.4.0...v6.0.0-alpha1[View commits]
 - Add experimental Vsphere module. {pull}4028[4028]
 - Add experimental Elasticsearch module. {pull}3903[3903]
 - Add experimental Kibana module. {pull}3895[3895]
+- Move elasticsearch metricset node_stats under node.stats namespace. {pull}4142[4142]
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -1956,20 +1956,12 @@ Node version.
 
 
 [float]
-== node_stats Fields
+== node.stats Fields
 
 node_stats
 
 
 
-[float]
-=== elasticsearch.node_stats.name
-
-type: keyword
-
-Node name.
-
-
 [float]
 == indices Fields
 
@@ -1978,31 +1970,31 @@ Node indices stats
 
 
 [float]
-=== elasticsearch.node_stats.indices.docs.count
+=== elasticsearch.node.stats.indices.docs.count
 
 type: long
 
 Total number of existing documents.
 
 
 [float]
-=== elasticsearch.node_stats.indices.docs.deleted
+=== elasticsearch.node.stats.indices.docs.deleted
 
 type: long
 
 Total number of deleted documents.
 
 
 [float]
-=== elasticsearch.node_stats.indices.segments.count
+=== elasticsearch.node.stats.indices.segments.count
 
 type: long
 
 Total number of segments.
 
 
 [float]
-=== elasticsearch.node_stats.indices.segments.memory.bytes
+=== elasticsearch.node.stats.indices.segments.memory.bytes
 
 type: long
 
@@ -2012,7 +2004,7 @@ Total size of segments in bytes.
 
 
 [float]
-=== elasticsearch.node_stats.indices.store.size.bytes
+=== elasticsearch.node.stats.indices.store.size.bytes
 
 type: long
 
@@ -2034,7 +2026,7 @@ Old memory pool stats.
 
 
 [float]
-=== elasticsearch.node_stats.jvm.mem.pools.old.max.bytes
+=== elasticsearch.node.stats.jvm.mem.pools.old.max.bytes
 
 type: long
 
@@ -2043,7 +2035,7 @@ format: bytes
 Max bytes.
 
 [float]
-=== elasticsearch.node_stats.jvm.mem.pools.old.peak.bytes
+=== elasticsearch.node.stats.jvm.mem.pools.old.peak.bytes
 
 type: long
 
@@ -2052,7 +2044,7 @@ format: bytes
 Peak bytes.
 
 [float]
-=== elasticsearch.node_stats.jvm.mem.pools.old.peak_max.bytes
+=== elasticsearch.node.stats.jvm.mem.pools.old.peak_max.bytes
 
 type: long
 
@@ -2061,7 +2053,7 @@ format: bytes
 Peak max bytes.
 
 [float]
-=== elasticsearch.node_stats.jvm.mem.pools.old.used.bytes
+=== elasticsearch.node.stats.jvm.mem.pools.old.used.bytes
 
 type: long
 
@@ -2077,7 +2069,7 @@ Young memory pool stats.
 
 
 [float]
-=== elasticsearch.node_stats.jvm.mem.pools.young.max.bytes
+=== elasticsearch.node.stats.jvm.mem.pools.young.max.bytes
 
 type: long
 
@@ -2086,7 +2078,7 @@ format: bytes
 Max bytes.
 
 [float]
-=== elasticsearch.node_stats.jvm.mem.pools.young.peak.bytes
+=== elasticsearch.node.stats.jvm.mem.pools.young.peak.bytes
 
 type: long
 
@@ -2095,7 +2087,7 @@ format: bytes
 Peak bytes.
 
 [float]
-=== elasticsearch.node_stats.jvm.mem.pools.young.peak_max.bytes
+=== elasticsearch.node.stats.jvm.mem.pools.young.peak_max.bytes
 
 type: long
 
@@ -2104,7 +2096,7 @@ format: bytes
 Peak max bytes.
 
 [float]
-=== elasticsearch.node_stats.jvm.mem.pools.young.used.bytes
+=== elasticsearch.node.stats.jvm.mem.pools.young.used.bytes
 
 type: long
 
@@ -2120,7 +2112,7 @@ Survivor memory pool stats.
 
 
 [float]
-=== elasticsearch.node_stats.jvm.mem.pools.survivor.max.bytes
+=== elasticsearch.node.stats.jvm.mem.pools.survivor.max.bytes
 
 type: long
 
@@ -2129,7 +2121,7 @@ format: bytes
 Max bytes.
 
 [float]
-=== elasticsearch.node_stats.jvm.mem.pools.survivor.peak.bytes
+=== elasticsearch.node.stats.jvm.mem.pools.survivor.peak.bytes
 
 type: long
 
@@ -2138,7 +2130,7 @@ format: bytes
 Peak bytes.
 
 [float]
-=== elasticsearch.node_stats.jvm.mem.pools.survivor.peak_max.bytes
+=== elasticsearch.node.stats.jvm.mem.pools.survivor.peak_max.bytes
 
 type: long
 
@@ -2147,7 +2139,7 @@ format: bytes
 Peak max bytes.
 
 [float]
-=== elasticsearch.node_stats.jvm.mem.pools.survivor.used.bytes
+=== elasticsearch.node.stats.jvm.mem.pools.survivor.used.bytes
 
 type: long
 
@@ -2170,14 +2162,14 @@ Old collection gc.
 
 
 [float]
-=== elasticsearch.node_stats.jvm.gc.collectors.old.collection.count
+=== elasticsearch.node.stats.jvm.gc.collectors.old.collection.count
 
 type: long
 
 
 
 [float]
-=== elasticsearch.node_stats.jvm.gc.collectors.old.collection.ms
+=== elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms
 
 type: long
 
@@ -2191,14 +2183,14 @@ Young collection gc.
 
 
 [float]
-=== elasticsearch.node_stats.jvm.gc.collectors.young.collection.count
+=== elasticsearch.node.stats.jvm.gc.collectors.young.collection.count
 
 type: long
 
 
 
 [float]
-=== elasticsearch.node_stats.jvm.gc.collectors.young.collection.ms
+=== elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms
 
 type: long
 

metricbeat/mb/mb.go

@@ -13,11 +13,17 @@ import (
 )
 
 const (
-	// ModuleData is the key used in events created by MetricSets to add data
+	// ModuleDataKey is the key used in events created by MetricSets to add data
 	// to an event that is common to the module. The data must be a
 	// common.MapStr and when the final event is built the object will be stored
 	// in the event under a key that is the module name.
-	ModuleData string = "_module"
+	ModuleDataKey string = "_module"
+
+	// NamespaceKey is used to define a different namespace for the metricset
+	// This is useful for dynamic metricsets or metricsets which do not
+	// put the name under the same name as the package. This is for example
+	// the case in elasticsearch `node_stats` which puts the data under `node.stats`.
+	NamespaceKey string = "_namespace"
 )
 
 // Module interfaces

metricbeat/mb/module/event.go

@@ -9,10 +9,6 @@ import (
 	"github.com/elastic/beats/metricbeat/mb"
 )
 
-const (
-	defaultType = "metricsets"
-)
-
 // EventBuilder is used for building MetricSet events. MetricSets generate a
 // data in the form of a common.MapStr. This builder transforms that data into
 // a complete event and applies any Module-level filtering.
@@ -38,8 +34,6 @@ func (b EventBuilder) Build() (common.MapStr, error) {
 	}
 
 	// Get and remove meta fields from the event created by the MetricSet.
-	indexName := getIndex(event, "")
-	typeName := getType(event, defaultType)
 	timestamp := getTimestamp(event, common.Time(b.StartTime))
 
 	// Apply filters.
@@ -49,56 +43,51 @@ func (b EventBuilder) Build() (common.MapStr, error) {
 		}
 	}
 
-	// Checks if additional meta information is provided by the MetricSet under the key ModuleData
-	// This is based on the convention that each MetricSet can provide module data under the key ModuleData
-	moduleData, moudleDataExists := event[mb.ModuleData]
-	if moudleDataExists {
-		delete(event, mb.ModuleData)
-	}
-
 	metricsetData := common.MapStr{
 		"module": b.ModuleName,
 		"name":   b.MetricSetName,
 	}
+	// Adds host name to event.
+	if b.Host != "" {
+		metricsetData["host"] = b.Host
+	}
 	if b.FetchDuration != 0 {
 		metricsetData["rtt"] = b.FetchDuration.Nanoseconds() / int64(time.Microsecond)
 	}
 
 	namespace := b.MetricSetName
 	if n, ok := event["_namespace"]; ok {
 		delete(event, "_namespace")
-		namespace = n.(string)
-		// TODO: check if namespace does not already exist
+		if ns, ok := n.(string); ok {
+			namespace = ns
+		}
+
 		metricsetData["namespace"] = namespace
 	}
 
-	event = common.MapStr{
-		"@timestamp":            timestamp,
-		"type":                  typeName,
-		common.EventMetadataKey: b.metadata,
-		b.ModuleName: common.MapStr{
-			namespace: event,
-		},
-		"metricset": metricsetData,
+	// Checks if additional meta information is provided by the MetricSet under the key ModuleData
+	// This is based on the convention that each MetricSet can provide module data under the key ModuleData
+	moduleData, moudleDataExists := event[mb.ModuleDataKey]
+	if moudleDataExists {
+		delete(event, mb.ModuleDataKey)
 	}
 
+	moduleEvent := common.MapStr{}
+	moduleEvent.Put(namespace, event)
+
 	// In case meta data exists, it is added on the module level
+	// This is mostly used for shared fields across multiple metricsets in one module
 	if moudleDataExists {
-		if _, ok := moduleData.(common.MapStr); ok {
-			event[b.ModuleName].(common.MapStr).Update(moduleData.(common.MapStr))
-		}
-	}
-
-	// Overwrite default index if set.
-	if indexName != "" {
-		event["beat"] = common.MapStr{
-			"index": indexName,
+		if data, ok := moduleData.(common.MapStr); ok {
+			moduleEvent.DeepUpdate(data)
 		}
 	}
 
-	// Adds host name to event.
-	if b.Host != "" {
-		event["metricset"].(common.MapStr)["host"] = b.Host
+	event = common.MapStr{
+		"@timestamp":            timestamp,
+		common.EventMetadataKey: b.metadata,
+		b.ModuleName:            moduleEvent,
+		"metricset":             metricsetData,
 	}
 
 	// Adds error to event in case error happened
@@ -111,33 +100,6 @@ func (b EventBuilder) Build() (common.MapStr, error) {
 	return event, nil
 }
 
-func getIndex(event common.MapStr, indexName string) string {
-	// Set index from event if set
-
-	if _, ok := event["index"]; ok {
-		indexName, ok = event["index"].(string)
-		if !ok {
-			logp.Err("Index couldn't be overwritten because event index is not string")
-		}
-		delete(event, "index")
-	}
-	return indexName
-}
-
-func getType(event common.MapStr, typeName string) string {
-
-	// Set type from event if set
-	if _, ok := event["type"]; ok {
-		typeName, ok = event["type"].(string)
-		if !ok {
-			logp.Err("Type couldn't be overwritten because event type is not string")
-		}
-		delete(event, "type")
-	}
-
-	return typeName
-}
-
 func getTimestamp(event common.MapStr, timestamp common.Time) common.Time {
 
 	// Set timestamp from event if set, move it to the top level

metricbeat/mb/module/event_test.go

@@ -45,7 +45,6 @@ func TestEventBuilder(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	assert.Equal(t, defaultType, event["type"])
 	assert.Equal(t, common.Time(startTime), event["@timestamp"])
 
 	metricset := event["metricset"].(common.MapStr)

metricbeat/mb/module/example_test.go

@@ -76,8 +76,7 @@ func ExampleWrapper() {
 	//     "module": "fake",
 	//     "name": "eventfetcher",
 	//     "rtt": 111
-	//   },
-	//   "type": "metricsets"
+	//   }
 	// }
 }
 

metricbeat/module/docker/cpu/data.go

@@ -15,7 +15,7 @@ func eventsMapping(cpuStatsList []CPUStats) []common.MapStr {
 
 func eventMapping(stats *CPUStats) common.MapStr {
 	event := common.MapStr{
-		mb.ModuleData: common.MapStr{
+		mb.ModuleDataKey: common.MapStr{
 			"container": stats.Container.ToMapStr(),
 		},
 		"core": stats.PerCpuUsage,