From eba8d787b0f633b7e46caf3b42067c234834db9a Mon Sep 17 00:00:00 2001 From: Thibaut Marmin Date: Fri, 5 Aug 2016 10:43:26 +0200 Subject: [PATCH] Broken logging (#115) * add logging * resolve timestamp pb to avoid marshall & unmarshall warnings * gofmt It closes #114 --- beater/dockerbeat.go | 35 +++++++++++----- config/config.go | 8 ++-- event/generator.go | 15 ++++--- event/generator_test.go | 88 ++++++++++++++++++++--------------------- 4 files changed, 84 insertions(+), 62 deletions(-) diff --git a/beater/dockerbeat.go b/beater/dockerbeat.go index 05502e7..4264e01 100644 --- a/beater/dockerbeat.go +++ b/beater/dockerbeat.go @@ -76,7 +76,7 @@ func (bt *Dockerbeat) Config(b *beat.Beat) error { err := cfgfile.Read(&bt.beatConfig, "") if err != nil { - logp.Err("Error reading configuration file: %v", err) + logp.Err("dockerbeat", "Error reading configuration file: %v", err) return err } @@ -143,7 +143,7 @@ func (bt *Dockerbeat) Config(b *beat.Beat) error { } logp.Info("dockerbeat", "Init dockerbeat") - logp.Info("dockerbeat", "Follow docker socket %q\n", bt.socketConfig.socket) + logp.Info("dockerbeat", "Follow docker socket %v\n", bt.socketConfig.socket) if bt.socketConfig.enableTls { logp.Info("dockerbeat", "TLS enabled\n") } else { @@ -193,7 +193,7 @@ func (bt *Dockerbeat) Setup(b *beat.Beat) error { } func (bt *Dockerbeat) Run(b *beat.Beat) error { - logp.Info("dockerbeat is running! Hit CTRL-C to stop it.") + logp.Info("dockerbeat", "dockerbeat is running! Hit CTRL-C to stop it.") var err error ticker := time.NewTicker(bt.period) @@ -209,7 +209,7 @@ func (bt *Dockerbeat) Run(b *beat.Beat) error { // check prerequisites err = bt.checkPrerequisites() if err != nil { - logp.Err("Unable to collect metrics: %v", err) + logp.Err("dockerbeat", "Unable to collect metrics: %v", err) bt.publishLogEvent(ERROR, fmt.Sprintf("Unable to collect metrics: %v", err)) continue } @@ -220,7 +220,7 @@ func (bt *Dockerbeat) Run(b *beat.Beat) error { duration := timerEnd.Sub(timerStart) if duration.Nanoseconds() > bt.period.Nanoseconds() { - logp.Warn("Ignoring tick(s) due to processing taking longer than one period") + logp.Warn("dockerbeat", "Ignoring tick(s) due to processing taking longer than one period") bt.publishLogEvent(WARN, "Ignoring tick(s) due to processing taking longer than one period") } } @@ -232,19 +232,21 @@ func (d *Dockerbeat) Cleanup(b *beat.Beat) error { func (d *Dockerbeat) Stop() { close(d.done) - logp.Info("Stopping dockerbeat") + logp.Info("dockerbeat", "Stopping dockerbeat") } func (d *Dockerbeat) RunOneTime(b *beat.Beat) error { + logp.Debug("dockerbeat", "Tick!, getting list of containers") containers, err := d.dockerClient.ListContainers(docker.ListContainersOptions{}) if err == nil { + logp.Debug("dockerbeat", "got %v containers", len(containers)) //export stats for each container for _, container := range containers { d.exportContainerStats(container) } } else { - logp.Err("Cannot get container list: %v", err) + logp.Err("dockerbeat", "Cannot get container list: %v", err) d.publishLogEvent(ERROR, fmt.Sprintf("Cannot get container list: %v", err)) } @@ -282,31 +284,46 @@ func (d *Dockerbeat) exportContainerStats(container docker.APIContainers) error // export events if it is enabled in the configuration if d.statsConfig.Container { + logp.Debug("dockerbeat", "generating container event for %v", container.ID) events = append(events, d.eventGenerator.GetContainerEvent(&container, stats)) + logp.Debug("dockerbeat", "container event append to event list (container %v)", container.ID) } if d.statsConfig.Cpu { + logp.Debug("dockerbeat", "generating cpu event for %v", container.ID) events = append(events, d.eventGenerator.GetCpuEvent(&container, stats)) + logp.Debug("dockerbeat", "container cpu append to event list (container %v)", container.ID) + } if d.statsConfig.Memory { + logp.Debug("dockerbeat", "generating memory event for %v", container.ID) events = append(events, d.eventGenerator.GetMemoryEvent(&container, stats)) + logp.Debug("dockerbeat", "container memory append to event list (container %v)", container.ID) + } if d.statsConfig.Blkio { + logp.Debug("dockerbeat", "generating blkio event for %v", container.ID) events = append(events, d.eventGenerator.GetBlkioEvent(&container, stats)) + logp.Debug("dockerbeat", "container blkio append to event list (container %v)", container.ID) + } if d.statsConfig.Net { + logp.Debug("dockerbeat", "generating net event for %v", container.ID) events = append(events, d.eventGenerator.GetNetworksEvent(&container, stats)...) + logp.Debug("dockerbeat", "container net append to event list (container %v)", container.ID) + } + logp.Info("dockerbeat", "Publishing %v events", len(events)) d.events.PublishEvents(events) } else if err == nil && stats == nil { - logp.Warn("Container was existing at listing but not when getting statistics: %v", container.ID) + logp.Warn("dockerbeat", "Container was existing at listing but not when getting statistics: %v", container.ID) d.publishLogEvent(WARN, fmt.Sprintf("Container was existing at listing but not when getting statistics: %v", container.ID)) } else { - logp.Err("An error occurred while getting docker stats: %v", err) + logp.Err("dockerbeat", "An error occurred while getting docker stats: %v", err) d.publishLogEvent(ERROR, fmt.Sprintf("An error occurred while getting docker stats: %v", err)) } }() diff --git a/config/config.go b/config/config.go index 602d263..faebe4a 100644 --- a/config/config.go +++ b/config/config.go @@ -23,8 +23,8 @@ type StatsConfig struct { } type DockerbeatConfig struct { - Period *int64 - Socket *string - Tls TlsConfig - Stats StatsConfig + Period *int64 `config:"period"` + Socket *string `config:"socket"` + Tls TlsConfig `config:"tls"` + Stats StatsConfig `config:"stats"` } diff --git a/event/generator.go b/event/generator.go index 25c601f..29e8ca5 100644 --- a/event/generator.go +++ b/event/generator.go @@ -2,6 +2,7 @@ package event import ( "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/fsouza/go-dockerclient" "github.com/ingensi/dockerbeat/calculator" "strings" @@ -33,6 +34,7 @@ type EventGenerator struct { } func (d *EventGenerator) GetContainerEvent(container *docker.APIContainers, stats *docker.Stats) common.MapStr { + logp.Debug("generator", "Generate container event %v", container.ID) event := common.MapStr{ "@timestamp": common.Time(stats.Read), "type": "container", @@ -43,7 +45,7 @@ func (d *EventGenerator) GetContainerEvent(container *docker.APIContainers, stat "container": common.MapStr{ "id": container.ID, "command": container.Command, - "created": time.Unix(container.Created, 0), + "created": common.Time(time.Unix(container.Created, 0)), "image": container.Image, "names": container.Names, "ports": d.convertContainerPorts(&container.Ports), @@ -56,7 +58,7 @@ func (d *EventGenerator) GetContainerEvent(container *docker.APIContainers, stat } func (d *EventGenerator) GetCpuEvent(container *docker.APIContainers, stats *docker.Stats) common.MapStr { - + logp.Debug("generator", "Generate cpu event %v", container.ID) calculator := d.CalculatorFactory.NewCPUCalculator( calculator.CPUData{ PerCpuUsage: stats.PreCPUStats.CPUUsage.PercpuUsage, @@ -91,6 +93,7 @@ func (d *EventGenerator) GetCpuEvent(container *docker.APIContainers, stats *doc } func (d *EventGenerator) GetNetworksEvent(container *docker.APIContainers, stats *docker.Stats) []common.MapStr { + logp.Debug("generator", "Generate network events %v", container.ID) events := []common.MapStr{} for netName, netStats := range stats.Networks { @@ -121,7 +124,7 @@ func (d *EventGenerator) GetNetworksEvent(container *docker.APIContainers, stats } func (d *EventGenerator) GetNetworkEvent(container *docker.APIContainers, time time.Time, network string, networkStats *docker.NetworkStats) common.MapStr { - + logp.Debug("generator", "Generate network event %v", container.ID) newNetworkData := calculator.NetworkData{ Time: time, RxBytes: networkStats.RxBytes, @@ -194,6 +197,7 @@ func (d *EventGenerator) GetNetworkEvent(container *docker.APIContainers, time t } func (d *EventGenerator) GetMemoryEvent(container *docker.APIContainers, stats *docker.Stats) common.MapStr { + logp.Debug("generator", "Generate memory event %v", container.ID) event := common.MapStr{ "@timestamp": common.Time(stats.Read), "type": "memory", @@ -216,6 +220,7 @@ func (d *EventGenerator) GetMemoryEvent(container *docker.APIContainers, stats * } func (d *EventGenerator) GetBlkioEvent(container *docker.APIContainers, stats *docker.Stats) common.MapStr { + logp.Debug("generator", "Generate blkio event %v", container.ID) blkioStats := d.buildStats(stats.Read, stats.BlkioStats.IOServicedRecursive) var event common.MapStr @@ -270,7 +275,7 @@ func (d *EventGenerator) GetBlkioEvent(container *docker.APIContainers, stats *d } func (d *EventGenerator) GetLogEvent(level string, message string) common.MapStr { - + logp.Debug("generator", "Generate log event with message: %v", message) event := common.MapStr{ "@timestamp": common.Time(time.Now()), "type": "log", @@ -301,7 +306,7 @@ func (d *EventGenerator) convertContainerPorts(ports *[]docker.APIPort) []map[st func (d *EventGenerator) CleanOldStats(containers []docker.APIContainers) { found := false d.NetworkStats.Lock() - for containerStatKey, _ := range d.NetworkStats.M { + for containerStatKey := range d.NetworkStats.M { for _, container := range containers { if container.ID == containerStatKey { found = true diff --git a/event/generator_test.go b/event/generator_test.go index 5804fa3..953de47 100644 --- a/event/generator_test.go +++ b/event/generator_test.go @@ -45,7 +45,7 @@ func TestEventGeneratorGetNetworksEventFirstPass(t *testing.T) { "container command", 9876543210, "Up", - []docker.APIPort{docker.APIPort{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, + []docker.APIPort{{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, 123, 456, []string{"/name1", "name1/fake"}, @@ -137,11 +137,11 @@ func TestEventGeneratorGetNetworksEventFirstPass(t *testing.T) { "containerID": container.ID, "containerName": "name1", "containerLabels": []common.MapStr{ - common.MapStr{ + { "key": "label1", "value": "value1", }, - common.MapStr{ + { "key": "label2", "value": "value2", }, @@ -164,11 +164,11 @@ func TestEventGeneratorGetNetworksEventFirstPass(t *testing.T) { "containerID": container.ID, "containerName": "name1", "containerLabels": []common.MapStr{ - common.MapStr{ + { "key": "label1", "value": "value1", }, - common.MapStr{ + { "key": "label2", "value": "value2", }, @@ -196,9 +196,9 @@ func TestEventGeneratorGetNetworksEventFirstPass(t *testing.T) { // check returned events assert.Equal(t, len(expectedEvents), 2) - for i, _ := range expectedEvents { + for i := range expectedEvents { checked := false - for j, _ := range events { + for j := range events { if equalEvent(expectedEvents[i], events[j]) { checked = true break @@ -248,7 +248,7 @@ func TestEventGeneratorGetNetworksEvent(t *testing.T) { "container command", 9876543210, "Up", - []docker.APIPort{docker.APIPort{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, + []docker.APIPort{{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, 123, 456, []string{"/name1", "name1/fake"}, @@ -352,11 +352,11 @@ func TestEventGeneratorGetNetworksEvent(t *testing.T) { "containerID": container.ID, "containerName": "name1", "containerLabels": []common.MapStr{ - common.MapStr{ + { "key": "label1", "value": "value1", }, - common.MapStr{ + { "key": "label2", "value": "value2", }, @@ -379,11 +379,11 @@ func TestEventGeneratorGetNetworksEvent(t *testing.T) { "containerID": container.ID, "containerName": "name1", "containerLabels": []common.MapStr{ - common.MapStr{ + { "key": "label1", "value": "value1", }, - common.MapStr{ + { "key": "label2", "value": "value2", }, @@ -411,9 +411,9 @@ func TestEventGeneratorGetNetworksEvent(t *testing.T) { // check returned events assert.Equal(t, len(expectedEvents), 2) - for i, _ := range expectedEvents { + for i := range expectedEvents { checked := false - for j, _ := range events { + for j := range events { if equalEvent(expectedEvents[i], events[j]) { checked = true break @@ -465,7 +465,7 @@ func TestEventGeneratorGetNetworksEventCleanSavedEvents(t *testing.T) { "container command", 9876543210, "Up", - []docker.APIPort{docker.APIPort{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, + []docker.APIPort{{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, 123, 456, []string{"/name1", "name1/fake"}, @@ -547,11 +547,11 @@ func TestEventGeneratorGetNetworksEventCleanSavedEvents(t *testing.T) { "containerID": container.ID, "containerName": "name1", "containerLabels": []common.MapStr{ - common.MapStr{ + { "key": "label1", "value": "value1", }, - common.MapStr{ + { "key": "label2", "value": "value2", }, @@ -614,7 +614,7 @@ func TestEventGeneratorGetContainerEvent(t *testing.T) { "container command", 9876543210, "Up", - []docker.APIPort{docker.APIPort{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, + []docker.APIPort{{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, 123, 456, []string{"/name1", "name1/fake"}, @@ -634,15 +634,15 @@ func TestEventGeneratorGetContainerEvent(t *testing.T) { "containerID": container.ID, "containerName": "name1", "containerLabels": []common.MapStr{ - common.MapStr{ + { "key": "label1", "value": "value1", }, - common.MapStr{ + { "key": "label2", "value": "value2", }, - common.MapStr{ + { "key": "label3_with_dots", "value": "value3", }, @@ -651,7 +651,7 @@ func TestEventGeneratorGetContainerEvent(t *testing.T) { "container": common.MapStr{ "id": container.ID, "command": container.Command, - "created": time.Unix(container.Created, 0), + "created": common.Time(time.Unix(container.Created, 0)), "image": container.Image, "names": container.Names, "ports": []map[string]interface{}{common.MapStr{ @@ -708,15 +708,15 @@ func TestEventGeneratorGetContainerEventWithNoPorts(t *testing.T) { "containerID": container.ID, "containerName": "name1", "containerLabels": []common.MapStr{ - common.MapStr{ + { "key": "label1", "value": "value1", }, - common.MapStr{ + { "key": "label2", "value": "value2", }, - common.MapStr{ + { "key": "label3_with_dots", "value": "value3", }, @@ -725,7 +725,7 @@ func TestEventGeneratorGetContainerEventWithNoPorts(t *testing.T) { "container": common.MapStr{ "id": container.ID, "command": container.Command, - "created": time.Unix(container.Created, 0), + "created": common.Time(time.Unix(container.Created, 0)), "image": container.Image, "names": container.Names, "ports": []map[string]interface{}{}, @@ -769,7 +769,7 @@ func TestEventGeneratorGetCpuEvent(t *testing.T) { "container command", 9876543210, "Up", - []docker.APIPort{docker.APIPort{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, + []docker.APIPort{{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, 123, 456, []string{"/name1", "name1/fake"}, @@ -816,11 +816,11 @@ func TestEventGeneratorGetCpuEvent(t *testing.T) { "containerID": container.ID, "containerName": "name1", "containerLabels": []common.MapStr{ - common.MapStr{ + { "key": "label1", "value": "value1", }, - common.MapStr{ + { "key": "label2", "value": "value2", }, @@ -868,7 +868,7 @@ func TestEventGeneratorGetMemoryEvent(t *testing.T) { "container command", 9876543210, "Up", - []docker.APIPort{docker.APIPort{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, + []docker.APIPort{{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, 123, 456, []string{"/name1", "name1/fake"}, @@ -886,11 +886,11 @@ func TestEventGeneratorGetMemoryEvent(t *testing.T) { "containerID": container.ID, "containerName": "name1", "containerLabels": []common.MapStr{ - common.MapStr{ + { "key": "label1", "value": "value1", }, - common.MapStr{ + { "key": "label2", "value": "value2", }, @@ -950,7 +950,7 @@ func TestEventGeneratorGetBlkioEventFirstPass(t *testing.T) { "container command", 9876543210, "Up", - []docker.APIPort{docker.APIPort{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, + []docker.APIPort{{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, 123, 456, []string{"name1", "name1/fake"}, @@ -983,11 +983,11 @@ func TestEventGeneratorGetBlkioEventFirstPass(t *testing.T) { "containerID": container.ID, "containerName": "name1", "containerLabels": []common.MapStr{ - common.MapStr{ + { "key": "label1", "value": "value1", }, - common.MapStr{ + { "key": "label2", "value": "value2", }, @@ -1045,7 +1045,7 @@ func TestEventGeneratorGetBlkioEvent(t *testing.T) { "container command", 9876543210, "Up", - []docker.APIPort{docker.APIPort{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, + []docker.APIPort{{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, 123, 456, []string{"/name1", "name1/fake"}, @@ -1086,11 +1086,11 @@ func TestEventGeneratorGetBlkioEvent(t *testing.T) { "containerID": container.ID, "containerName": "name1", "containerLabels": []common.MapStr{ - common.MapStr{ + { "key": "label1", "value": "value1", }, - common.MapStr{ + { "key": "label2", "value": "value2", }, @@ -1150,7 +1150,7 @@ func TestEventGeneratorGetBlkioEventCleanSavedEvents(t *testing.T) { "container command", 9876543210, "Up", - []docker.APIPort{docker.APIPort{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, + []docker.APIPort{{PrivatePort: 1234, PublicPort: 4567, Type: "portType", IP: "123.456.879.1"}}, 123, 456, []string{"/name1", "name1/fake"}, @@ -1198,11 +1198,11 @@ func TestEventGeneratorGetBlkioEventCleanSavedEvents(t *testing.T) { "containerID": container.ID, "containerName": "name1", "containerLabels": []common.MapStr{ - common.MapStr{ + { "key": "label1", "value": "value1", }, - common.MapStr{ + { "key": "label2", "value": "value2", }, @@ -1451,9 +1451,9 @@ func getBlkioStats(read time.Time, reads uint64, writes uint64, total uint64) do Read: read, BlkioStats: blkioStats{ IOServicedRecursive: []docker.BlkioStatsEntry{ - docker.BlkioStatsEntry{Major: 0, Minor: 0, Op: "Read", Value: reads}, - docker.BlkioStatsEntry{Major: 0, Minor: 0, Op: "Write", Value: writes}, - docker.BlkioStatsEntry{Major: 0, Minor: 0, Op: "Total", Value: total}, + {Major: 0, Minor: 0, Op: "Read", Value: reads}, + {Major: 0, Minor: 0, Op: "Write", Value: writes}, + {Major: 0, Minor: 0, Op: "Total", Value: total}, }, }, }