From 15db49f2650bdf9b9b6723d49fa3449698f65ae7 Mon Sep 17 00:00:00 2001 From: Douae Jeouit Date: Fri, 3 Feb 2017 11:53:17 +0100 Subject: [PATCH] Metricbeat : new metricset image for docker module (#3467) --- CHANGELOG.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 61 ++++++++++++++++++ metricbeat/docs/modules/docker.asciidoc | 4 ++ metricbeat/docs/modules/docker/image.asciidoc | 19 ++++++ metricbeat/include/list.go | 1 + metricbeat/metricbeat.template-es2x.json | 31 +++++++++ metricbeat/metricbeat.template.json | 29 +++++++++ .../module/docker/image/_meta/data.json | 19 ++++++ .../module/docker/image/_meta/docs.asciidoc | 3 + .../module/docker/image/_meta/fields.yml | 41 ++++++++++++ metricbeat/module/docker/image/data.go | 37 +++++++++++ metricbeat/module/docker/image/image.go | 63 +++++++++++++++++++ .../docker/image/image_integration_test.go | 23 +++++++ metricbeat/tests/system/test_docker.py | 31 +++++++++ 14 files changed, 363 insertions(+) create mode 100644 metricbeat/docs/modules/docker/image.asciidoc create mode 100644 metricbeat/module/docker/image/_meta/data.json create mode 100644 metricbeat/module/docker/image/_meta/docs.asciidoc create mode 100644 metricbeat/module/docker/image/_meta/fields.yml create mode 100644 metricbeat/module/docker/image/data.go create mode 100644 metricbeat/module/docker/image/image.go create mode 100644 metricbeat/module/docker/image/image_integration_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 06978e0c0107..10dce15ced1a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -92,6 +92,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Kafka consumer groups metricset. {pull}3240[3240] - Add dynamic configuration reloading for modules. {pull}3281[3281] - Add docker health metricset {pull}3357[3357] +- Add docker image metricset {pull}3467[3467] - System module uses new matchers for white-listing processes. {pull}3469[3469] - Add CEPH module with health metricset. {pull}3311[3311] - Add php_fpm module with pool metricset. {pull}3415[3415] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d1121ca37c99..a80b8042c0a6 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1399,6 +1399,67 @@ type: integer Healthcheck status code +[float] +== image Fields + +Docker image metrics. + + + +[float] +== id Fields + +The image layers identifier. + + + +[float] +=== docker.image.id.current + +type: keyword + +Unique image identifier given upon its creation. + + +[float] +=== docker.image.id.parent + +type: keyword + +Identifier of the image, if it exists, from which the current image directly descends. + + +[float] +=== docker.image.created + +type: date + +Date and time when the image was created. + + +[float] +== size Fields + +Image size layers. + + + +[float] +=== docker.image.size.virtual + +type: long + +Size of the image. + + +[float] +=== docker.image.size.regular + +type: long + +Total size of the all cached images associated to the current image. + + [float] == info Fields diff --git a/metricbeat/docs/modules/docker.asciidoc b/metricbeat/docs/modules/docker.asciidoc index 2388f6ea9e0b..2d11ff9b87b1 100644 --- a/metricbeat/docs/modules/docker.asciidoc +++ b/metricbeat/docs/modules/docker.asciidoc @@ -46,6 +46,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -60,6 +62,8 @@ include::docker/diskio.asciidoc[] include::docker/healthcheck.asciidoc[] +include::docker/image.asciidoc[] + include::docker/info.asciidoc[] include::docker/memory.asciidoc[] diff --git a/metricbeat/docs/modules/docker/image.asciidoc b/metricbeat/docs/modules/docker/image.asciidoc new file mode 100644 index 000000000000..2771f9273325 --- /dev/null +++ b/metricbeat/docs/modules/docker/image.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-docker-image]] +include::../../../module/docker/image/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/image/_meta/data.json[] +---- diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 4e3b8d4a658b..c80c67e525c6 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -22,6 +22,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/docker/cpu" _ "github.com/elastic/beats/metricbeat/module/docker/diskio" _ "github.com/elastic/beats/metricbeat/module/docker/healthcheck" + _ "github.com/elastic/beats/metricbeat/module/docker/image" _ "github.com/elastic/beats/metricbeat/module/docker/info" _ "github.com/elastic/beats/metricbeat/module/docker/memory" _ "github.com/elastic/beats/metricbeat/module/docker/network" diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index 83a3fa5c6ccb..e49291cbc95c 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -815,6 +815,37 @@ } } }, + "image": { + "properties": { + "created": { + "type": "date" + }, + "id": { + "properties": { + "current": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "parent": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + }, + "size": { + "properties": { + "regular": { + "type": "long" + }, + "virtual": { + "type": "long" + } + } + } + } + }, "info": { "properties": { "containers": { diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index f317f31acd31..8286409179e1 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -811,6 +811,35 @@ } } }, + "image": { + "properties": { + "created": { + "type": "date" + }, + "id": { + "properties": { + "current": { + "ignore_above": 1024, + "type": "keyword" + }, + "parent": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "size": { + "properties": { + "regular": { + "type": "long" + }, + "virtual": { + "type": "long" + } + } + } + } + }, "info": { "properties": { "containers": { diff --git a/metricbeat/module/docker/image/_meta/data.json b/metricbeat/module/docker/image/_meta/data.json new file mode 100644 index 000000000000..0c3fadc2f75d --- /dev/null +++ b/metricbeat/module/docker/image/_meta/data.json @@ -0,0 +1,19 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"docker", + "name":"image", + "rtt":44269 + }, + "docker":{ + "image":{ + "example": "image" + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/docker/image/_meta/docs.asciidoc b/metricbeat/module/docker/image/_meta/docs.asciidoc new file mode 100644 index 000000000000..d78ba7a3d228 --- /dev/null +++ b/metricbeat/module/docker/image/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== docker image MetricSet + +This is the image metricset of the module docker. diff --git a/metricbeat/module/docker/image/_meta/fields.yml b/metricbeat/module/docker/image/_meta/fields.yml new file mode 100644 index 000000000000..c4bb774a38fe --- /dev/null +++ b/metricbeat/module/docker/image/_meta/fields.yml @@ -0,0 +1,41 @@ +- name: image + type: group + description: > + Docker image metrics. + fields: + - name: id + type: group + description: > + The image layers identifier. + fields: + - name: current + type: keyword + description: > + Unique image identifier given upon its creation. + - name: parent + type: keyword + description: > + Identifier of the image, if it exists, from which the current image directly descends. + - name: created + type: date + description: > + Date and time when the image was created. + - name: size + type: group + description: > + Image size layers. + fields: + - name: virtual + type: long + description: > + Size of the image. + - name: regular + type: long + description: > + Total size of the all cached images associated to the current image. + +# TODO : How to describe tags & labels list ? +# - name: tags +# type: list ? +# description: > +# Descriptive or given name(s) to the image. diff --git a/metricbeat/module/docker/image/data.go b/metricbeat/module/docker/image/data.go new file mode 100644 index 000000000000..877a3d7db639 --- /dev/null +++ b/metricbeat/module/docker/image/data.go @@ -0,0 +1,37 @@ +package image + +import ( + "github.com/elastic/beats/libbeat/common" + + "github.com/elastic/beats/metricbeat/module/docker" + dc "github.com/fsouza/go-dockerclient" + "time" +) + +func eventsMapping(imagesList []dc.APIImages) []common.MapStr { + events := []common.MapStr{} + for _, image := range imagesList { + events = append(events, eventMapping(&image)) + } + return events +} + +func eventMapping(image *dc.APIImages) common.MapStr { + event := common.MapStr{ + "id": common.MapStr{ + "current": image.ID, + "parent": image.ParentID, + }, + "created": common.Time(time.Unix(image.Created, 0)), + "size": common.MapStr{ + "regular": image.Size, + "virtual": image.VirtualSize, + }, + "tags": image.RepoTags, + } + labels := docker.DeDotLabels(image.Labels) + if len(labels) > 0 { + event["labels"] = labels + } + return event +} diff --git a/metricbeat/module/docker/image/image.go b/metricbeat/module/docker/image/image.go new file mode 100644 index 000000000000..ebac45441924 --- /dev/null +++ b/metricbeat/module/docker/image/image.go @@ -0,0 +1,63 @@ +package image + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/docker" + + dc "github.com/fsouza/go-dockerclient" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + if err := mb.Registry.AddMetricSet("docker", "image", New); err != nil { + panic(err) + } +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + dockerClient *dc.Client +} + +// New create a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + + logp.Warn("EXPERIMENTAL: The docker info metricset is experimental") + + config := docker.Config{} + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + client, err := docker.NewDockerClient(base.HostData().URI, config) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + dockerClient: client, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch() ([]common.MapStr, error) { + + images, err := m.dockerClient.ListImages(dc.ListImagesOptions{}) + if err != nil { + return nil, err + } + + return eventsMapping(images), nil +} diff --git a/metricbeat/module/docker/image/image_integration_test.go b/metricbeat/module/docker/image/image_integration_test.go new file mode 100644 index 000000000000..988927ab852e --- /dev/null +++ b/metricbeat/module/docker/image/image_integration_test.go @@ -0,0 +1,23 @@ +package image + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewEventsFetcher(t, getConfig()) + err := mbtest.WriteEvents(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "docker", + "metricsets": []string{"image"}, + "hosts": []string{"unix:///var/run/docker.sock"}, + } +} diff --git a/metricbeat/tests/system/test_docker.py b/metricbeat/tests/system/test_docker.py index 1aaff6c5b758..be883f322032 100644 --- a/metricbeat/tests/system/test_docker.py +++ b/metricbeat/tests/system/test_docker.py @@ -192,6 +192,37 @@ def test_health_fields(self): evt = self.remove_labels(evt) self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_image_fields(self): + """ + test image fields + """ + self.render_config_template(modules=[{ + "name": "docker", + "metricsets": ["image"], + "hosts": ["unix:///var/run/docker.sock"], + "period": "1s", + }]) + + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + proc.check_kill_and_wait() + + # Ensure no errors or warnings exist in the log. + log = self.get_log() + self.assertNotRegexpMatches(log.replace("WARN EXPERIMENTAL", ""), "ERR|WARN") + + output = self.read_output_json() + evt = output[0] + + if 'tags' in evt["docker"]["image"] : + del evt["docker"]["image"]["tags"] + + if 'labels' in evt["docker"]["image"] : + del evt["docker"]["image"]["labels"] + + self.assert_fields_are_documented(evt) + def remove_labels(self, evt): if 'labels' in evt["docker"]["container"]: