Metricbeat : new metricset image for docker module (#3467)

CHANGELOG.asciidoc

@@ -92,6 +92,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Kafka consumer groups metricset. {pull}3240[3240]
 - Add dynamic configuration reloading for modules. {pull}3281[3281]
 - Add docker health metricset {pull}3357[3357]
+- Add docker image metricset {pull}3467[3467]
 - System module uses new matchers for white-listing processes. {pull}3469[3469]
 - Add CEPH module with health metricset. {pull}3311[3311]
 - Add php_fpm module with pool metricset. {pull}3415[3415]

metricbeat/docs/fields.asciidoc

@@ -1399,6 +1399,67 @@ type: integer
 Healthcheck status code
 
 
+[float]
+== image Fields
+
+Docker image metrics.
+
+
+
+[float]
+== id Fields
+
+The image layers identifier.
+
+
+
+[float]
+=== docker.image.id.current
+
+type: keyword
+
+Unique image identifier given upon its creation.
+
+
+[float]
+=== docker.image.id.parent
+
+type: keyword
+
+Identifier of the image, if it exists, from which the current image directly descends.
+
+
+[float]
+=== docker.image.created
+
+type: date
+
+Date and time when the image was created.
+
+
+[float]
+== size Fields
+
+Image size layers.
+
+
+
+[float]
+=== docker.image.size.virtual
+
+type: long
+
+Size of the image.
+
+
+[float]
+=== docker.image.size.regular
+
+type: long
+
+Total size of the all cached images associated to the current image.
+
+
 [float]
 == info Fields
 

metricbeat/docs/modules/docker.asciidoc

@@ -46,6 +46,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-docker-healthcheck,healthcheck>>
 
+* <<metricbeat-metricset-docker-image,image>>
+
 * <<metricbeat-metricset-docker-info,info>>
 
 * <<metricbeat-metricset-docker-memory,memory>>
@@ -60,6 +62,8 @@ include::docker/diskio.asciidoc[]
 
 include::docker/healthcheck.asciidoc[]
 
+include::docker/image.asciidoc[]
+
 include::docker/info.asciidoc[]
 
 include::docker/memory.asciidoc[]

metricbeat/docs/modules/docker/image.asciidoc

@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-docker-image]]
+include::../../../module/docker/image/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-docker,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/docker/image/_meta/data.json[]
+----

metricbeat/include/list.go

@@ -22,6 +22,7 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/docker/cpu"
 	_ "github.com/elastic/beats/metricbeat/module/docker/diskio"
 	_ "github.com/elastic/beats/metricbeat/module/docker/healthcheck"
+	_ "github.com/elastic/beats/metricbeat/module/docker/image"
 	_ "github.com/elastic/beats/metricbeat/module/docker/info"
 	_ "github.com/elastic/beats/metricbeat/module/docker/memory"
 	_ "github.com/elastic/beats/metricbeat/module/docker/network"

metricbeat/metricbeat.template-es2x.json

@@ -815,6 +815,37 @@
                 }
               }
             },
+            "image": {
+              "properties": {
+                "created": {
+                  "type": "date"
+                },
+                "id": {
+                  "properties": {
+                    "current": {
+                      "ignore_above": 1024,
+                      "index": "not_analyzed",
+                      "type": "string"
+                    },
+                    "parent": {
+                      "ignore_above": 1024,
+                      "index": "not_analyzed",
+                      "type": "string"
+                    }
+                  }
+                },
+                "size": {
+                  "properties": {
+                    "regular": {
+                      "type": "long"
+                    },
+                    "virtual": {
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            },
             "info": {
               "properties": {
                 "containers": {

metricbeat/metricbeat.template.json

@@ -811,6 +811,35 @@
                 }
               }
             },
+            "image": {
+              "properties": {
+                "created": {
+                  "type": "date"
+                },
+                "id": {
+                  "properties": {
+                    "current": {
+                      "ignore_above": 1024,
+                      "type": "keyword"
+                    },
+                    "parent": {
+                      "ignore_above": 1024,
+                      "type": "keyword"
+                    }
+                  }
+                },
+                "size": {
+                  "properties": {
+                    "regular": {
+                      "type": "long"
+                    },
+                    "virtual": {
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            },
             "info": {
               "properties": {
                 "containers": {

metricbeat/module/docker/image/_meta/data.json

@@ -0,0 +1,19 @@
+{
+    "@timestamp":"2016-05-23T08:05:34.853Z",
+    "beat":{
+        "hostname":"beathost",
+        "name":"beathost"
+    },
+    "metricset":{
+        "host":"localhost",
+        "module":"docker",
+        "name":"image",
+        "rtt":44269
+    },
+    "docker":{
+        "image":{
+            "example": "image"
+        }
+    },
+    "type":"metricsets"
+}

metricbeat/module/docker/image/_meta/docs.asciidoc

@@ -0,0 +1,3 @@
+=== docker image MetricSet
+
+This is the image metricset of the module docker.

metricbeat/module/docker/image/_meta/fields.yml

@@ -0,0 +1,41 @@
+- name: image
+  type: group
+  description: >
+    Docker image metrics.
+  fields:
+    - name: id
+      type: group
+      description: >
+        The image layers identifier.
+      fields:
+        - name: current
+          type: keyword
+          description: >
+             Unique image identifier given upon its creation.
+        - name: parent
+          type: keyword
+          description: >
+             Identifier of the image, if it exists, from which the current image directly descends.
+    - name: created
+      type: date
+      description: >
+        Date and time when the image was created.
+    - name: size
+      type: group
+      description: >
+        Image size layers.
+      fields:
+        - name: virtual
+          type: long
+          description: >
+             Size of the image.
+        - name: regular
+          type: long
+          description: >
+             Total size of the all cached images associated to the current image.
+
+# TODO : How to describe tags & labels list ?
+#    - name: tags
+#      type: list ?
+#      description: >
+#        Descriptive or given name(s) to the image.

metricbeat/module/docker/image/data.go

@@ -0,0 +1,37 @@
+package image
+
+import (
+	"github.com/elastic/beats/libbeat/common"
+
+	"github.com/elastic/beats/metricbeat/module/docker"
+	dc "github.com/fsouza/go-dockerclient"
+	"time"
+)
+
+func eventsMapping(imagesList []dc.APIImages) []common.MapStr {
+	events := []common.MapStr{}
+	for _, image := range imagesList {
+		events = append(events, eventMapping(&image))
+	}
+	return events
+}
+
+func eventMapping(image *dc.APIImages) common.MapStr {
+	event := common.MapStr{
+		"id": common.MapStr{
+			"current": image.ID,
+			"parent":  image.ParentID,
+		},
+		"created": common.Time(time.Unix(image.Created, 0)),
+		"size": common.MapStr{
+			"regular": image.Size,
+			"virtual": image.VirtualSize,
+		},
+		"tags": image.RepoTags,
+	}
+	labels := docker.DeDotLabels(image.Labels)
+	if len(labels) > 0 {
+		event["labels"] = labels
+	}
+	return event
+}

metricbeat/module/docker/image/image.go

@@ -0,0 +1,63 @@
+package image
+
+import (
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/logp"
+	"github.com/elastic/beats/metricbeat/mb"
+	"github.com/elastic/beats/metricbeat/module/docker"
+
+	dc "github.com/fsouza/go-dockerclient"
+)
+
+// init registers the MetricSet with the central registry.
+// The New method will be called after the setup of the module and before starting to fetch data
+func init() {
+	if err := mb.Registry.AddMetricSet("docker", "image", New); err != nil {
+		panic(err)
+	}
+}
+
+// MetricSet type defines all fields of the MetricSet
+// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with
+// additional entries. These variables can be used to persist data or configuration between
+// multiple fetch calls.
+type MetricSet struct {
+	mb.BaseMetricSet
+	dockerClient *dc.Client
+}
+
+// New create a new instance of the MetricSet
+// Part of new is also setting up the configuration by processing additional
+// configuration entries if needed.
+func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
+
+	logp.Warn("EXPERIMENTAL: The docker info metricset is experimental")
+
+	config := docker.Config{}
+	if err := base.Module().UnpackConfig(&config); err != nil {
+		return nil, err
+	}
+
+	client, err := docker.NewDockerClient(base.HostData().URI, config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &MetricSet{
+		BaseMetricSet: base,
+		dockerClient:  client,
+	}, nil
+}
+
+// Fetch methods implements the data gathering and data conversion to the right format
+// It returns the event which is then forward to the output. In case of an error, a
+// descriptive error must be returned.
+func (m *MetricSet) Fetch() ([]common.MapStr, error) {
+
+	images, err := m.dockerClient.ListImages(dc.ListImagesOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return eventsMapping(images), nil
+}

metricbeat/module/docker/image/image_integration_test.go

@@ -0,0 +1,23 @@
+package image
+
+import (
+	"testing"
+
+	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+)
+
+func TestData(t *testing.T) {
+	f := mbtest.NewEventsFetcher(t, getConfig())
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
+func getConfig() map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "docker",
+		"metricsets": []string{"image"},
+		"hosts":      []string{"unix:///var/run/docker.sock"},
+	}
+}