From 67952f17f51b166b420b078fc9384e45b39f5c67 Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Wed, 24 Aug 2016 22:41:53 +0300 Subject: [PATCH 01/18] Add dmcache input plugin --- plugins/inputs/all/all.go | 1 + plugins/inputs/dmcache/README.md | 11 ++ plugins/inputs/dmcache/dmcache.go | 49 ++++++++ plugins/inputs/dmcache/dmcache_linux.go | 131 +++++++++++++++++++++ plugins/inputs/dmcache/dmcache_notlinux.go | 11 ++ plugins/inputs/dmcache/dmcache_test.go | 94 +++++++++++++++ 6 files changed, 297 insertions(+) create mode 100644 plugins/inputs/dmcache/README.md create mode 100644 plugins/inputs/dmcache/dmcache.go create mode 100644 plugins/inputs/dmcache/dmcache_linux.go create mode 100644 plugins/inputs/dmcache/dmcache_notlinux.go create mode 100644 plugins/inputs/dmcache/dmcache_test.go diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index af759aac8ba4e..c7ac4762b476d 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -14,6 +14,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/couchbase" _ "github.com/influxdata/telegraf/plugins/inputs/couchdb" _ "github.com/influxdata/telegraf/plugins/inputs/disque" + _ "github.com/influxdata/telegraf/plugins/inputs/dmcache" _ "github.com/influxdata/telegraf/plugins/inputs/dns_query" _ "github.com/influxdata/telegraf/plugins/inputs/docker" _ "github.com/influxdata/telegraf/plugins/inputs/dovecot" diff --git a/plugins/inputs/dmcache/README.md b/plugins/inputs/dmcache/README.md new file mode 100644 index 0000000000000..87931376e0e18 --- /dev/null +++ b/plugins/inputs/dmcache/README.md @@ -0,0 +1,11 @@ +# DMCache Input Plugin + +This plugin provide a native collection for dmsetup based statistics for dm-cache + +## Configuration + +``` +[[inputs.dmcache]] +## Whether to report per-device stats or not + per_device = true +``` diff --git a/plugins/inputs/dmcache/dmcache.go b/plugins/inputs/dmcache/dmcache.go new file mode 100644 index 0000000000000..bfcb6dadcb751 --- /dev/null +++ b/plugins/inputs/dmcache/dmcache.go @@ -0,0 +1,49 @@ +package dmcache + +import ( + "os/exec" + "strings" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +type DMCache struct { + PerDevice bool `toml:"per_device"` + rawStatus func() ([]string, error) +} + +var sampleConfig = ` + ## Whether to report per-device stats or not + per_device = true +` + +func (c *DMCache) SampleConfig() string { + return sampleConfig +} + +func (c *DMCache) Description() string { + return "Provide a native collection for dmsetup based statistics for dm-cache" +} + +func init() { + inputs.Add("dmcache", func() telegraf.Input { + return &DMCache{ + PerDevice: true, + rawStatus: func() ([]string, error) { + out, err := exec.Command("/bin/sh", "-c", "sudo /sbin/dmsetup status --target cache").Output() + if err != nil { + return nil, err + } + if string(out) == "No devices found\n" { + return nil, nil + } + + status := strings.Split(string(out), "\n") + status = status[:len(status)-1] // removing last empty line + + return status, nil + }, + } + }) +} diff --git a/plugins/inputs/dmcache/dmcache_linux.go b/plugins/inputs/dmcache/dmcache_linux.go new file mode 100644 index 0000000000000..ed8b88c6809d3 --- /dev/null +++ b/plugins/inputs/dmcache/dmcache_linux.go @@ -0,0 +1,131 @@ +// +build linux + +package dmcache + +import ( + "strconv" + "strings" + + "github.com/influxdata/telegraf" +) + +const metricName = "dmcache" + +var fieldNames = [...]string{ + "metadata_used", + "metadata_free", + "cache_used", + "cache_free", + "read_hits", + "read_misses", + "write_hits", + "write_misses", + "demotions", + "promotions", + "dirty", +} + +func (c *DMCache) Gather(acc telegraf.Accumulator) error { + outputLines, err := c.rawStatus() + if err != nil { + return err + } + + var total map[string]interface{} + if !c.PerDevice { + total = make(map[string]interface{}) + } + + for _, s := range outputLines { + fields := make(map[string]interface{}) + data, err := parseDMSetupStatus(s) + if err != nil { + return err + } + + for _, f := range fieldNames { + fields[f] = calculateSize(data, f) + } + + if c.PerDevice { + tags := map[string]string{"device": data["device"].(string)} + acc.AddFields(metricName, fields, tags) + } else { + aggregateStats(total, fields) + } + } + + if !c.PerDevice { + acc.AddFields(metricName, total, nil) + } + + return nil +} + +func parseDMSetupStatus(line string) (status map[string]interface{}, err error) { + defer func() { + if r := recover(); r != nil { + status = nil + err = r.(error) + } + }() + + values := strings.Split(line, " ") + status = make(map[string]interface{}) + + status["device"] = values[0][:len(values[0])-1] + status["length"] = toInt(values[2]) + status["target"] = values[3] + status["metadata_blocksize"] = toInt(values[4]) + status["metadata_used"] = toInt(strings.Split(values[5], "/")[0]) + status["metadata_total"] = toInt(strings.Split(values[5], "/")[1]) + status["cache_blocksize"] = toInt(values[6]) + status["cache_used"] = toInt(strings.Split(values[7], "/")[0]) + status["cache_total"] = toInt(strings.Split(values[7], "/")[1]) + status["read_hits"] = toInt(values[8]) + status["read_misses"] = toInt(values[9]) + status["write_hits"] = toInt(values[10]) + status["write_misses"] = toInt(values[11]) + status["demotions"] = toInt(values[12]) + status["promotions"] = toInt(values[13]) + status["dirty"] = toInt(values[14]) + status["blocksize"] = 512 + + return status, nil +} + +func toInt(s string) int { + i, err := strconv.Atoi(s) + if err != nil { + panic(err) + } + return i +} + +func calculateSize(data map[string]interface{}, key string) (value int) { + if key == "metadata_free" { + value = data["metadata_total"].(int) - data["metadata_used"].(int) + } else if key == "cache_free" { + value = data["cache_total"].(int) - data["cache_used"].(int) - data["dirty"].(int) + } else { + value = data[key].(int) + } + + if key == "metadata_free" || key == "metadata_used" { + value = value * data["blocksize"].(int) * data["metadata_blocksize"].(int) + } else { + value = value * data["blocksize"].(int) * data["cache_blocksize"].(int) + } + + return +} + +func aggregateStats(total, fields map[string]interface{}) { + for key, value := range fields { + if _, ok := total[key]; ok { + total[key] = total[key].(int) + value.(int) + } else { + total[key] = value.(int) + } + } +} diff --git a/plugins/inputs/dmcache/dmcache_notlinux.go b/plugins/inputs/dmcache/dmcache_notlinux.go new file mode 100644 index 0000000000000..478c2f163e4dc --- /dev/null +++ b/plugins/inputs/dmcache/dmcache_notlinux.go @@ -0,0 +1,11 @@ +// +build !linux + +package dmcache + +import ( + "github.com/influxdata/telegraf" +) + +func (c *DMCache) Gather(acc telegraf.Accumulator) error { + return nil +} diff --git a/plugins/inputs/dmcache/dmcache_test.go b/plugins/inputs/dmcache/dmcache_test.go new file mode 100644 index 0000000000000..42d5072990802 --- /dev/null +++ b/plugins/inputs/dmcache/dmcache_test.go @@ -0,0 +1,94 @@ +// +build linux + +package dmcache + +import ( + "testing" + + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func output2Devices() ([]string, error) { + return []string{ + "cs-1: 0 4883791872 cache 8 1018/1501122 512 7/464962 139 352643 15 46 0 7 0 1 writeback 2 migration_threshold 2048 mq 10 random_threshold 4 sequential_threshold 512 discard_promote_adjustment 1 read_promote_adjustment 4 write_promote_adjustment 8", + "cs-2: 0 4294967296 cache 8 72352/1310720 128 26/24327168 2409 286 265 524682 0 0 0 1 writethrough 2 migration_threshold 2048 mq 10 random_threshold 4 sequential_threshold 512 discard_promote_adjustment 1 read_promote_adjustment 4 write_promote_adjustment 8", + }, nil +} + +var dmc1 = &DMCache{ + PerDevice: true, + rawStatus: output2Devices, +} + +func TestDMCacheStats_1(t *testing.T) { + var acc testutil.Accumulator + + err := dmc1.Gather(&acc) + require.NoError(t, err) + + tags1 := map[string]string{ + "device": "cs-1", + } + fields1 := map[string]interface{}{ + "metadata_used": 4169728, + "metadata_free": 6144425984, + "cache_used": 1835008, + "cache_free": 121885163520, + "read_hits": 36438016, + "read_misses": 92443246592, + "write_hits": 3932160, + "write_misses": 12058624, + "demotions": 0, + "promotions": 1835008, + "dirty": 0, + } + acc.AssertContainsTaggedFields(t, "dmcache", fields1, tags1) + + tags2 := map[string]string{ + "device": "cs-2", + } + fields2 := map[string]interface{}{ + "metadata_used": 296353792, + "metadata_free": 5072355328, + "cache_used": 1703936, + "cache_free": 1594303578112, + "read_hits": 157876224, + "read_misses": 18743296, + "write_hits": 17367040, + "write_misses": 34385559552, + "demotions": 0, + "promotions": 0, + "dirty": 0, + } + acc.AssertContainsTaggedFields(t, "dmcache", fields2, tags2) +} + +var dmc2 = &DMCache{ + PerDevice: false, + rawStatus: output2Devices, +} + +func TestDMCacheStats_2(t *testing.T) { + var acc testutil.Accumulator + + err := dmc2.Gather(&acc) + require.NoError(t, err) + + tags := map[string]string{} + + fields := map[string]interface{}{ + "metadata_used": 300523520, + "metadata_free": 11216781312, + "cache_used": 3538944, + "cache_free": 1716188741632, + "read_hits": 194314240, + "read_misses": 92461989888, + "write_hits": 21299200, + "write_misses": 34397618176, + "demotions": 0, + "promotions": 1835008, + "dirty": 0, + } + acc.AssertContainsTaggedFields(t, "dmcache", fields, tags) +} From 7143b600c8f70c729007659fe01381075783126e Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Wed, 29 Mar 2017 12:19:25 +0300 Subject: [PATCH 02/18] dmcache refactoring --- plugins/inputs/dmcache/README.md | 10 ++- plugins/inputs/dmcache/dmcache.go | 4 +- plugins/inputs/dmcache/dmcache_linux.go | 109 +++++++++++++++--------- plugins/inputs/dmcache/dmcache_test.go | 25 +++++- 4 files changed, 99 insertions(+), 49 deletions(-) diff --git a/plugins/inputs/dmcache/README.md b/plugins/inputs/dmcache/README.md index 87931376e0e18..ae4142e9aa2fa 100644 --- a/plugins/inputs/dmcache/README.md +++ b/plugins/inputs/dmcache/README.md @@ -1,11 +1,15 @@ # DMCache Input Plugin -This plugin provide a native collection for dmsetup based statistics for dm-cache +This plugin provide a native collection for dmsetup based statistics for dm-cache. + +This plugin requires sudo, that is why you should setup and be sure that the telegraf is able to execute sudo without a password. + +`sudo /sbin/dmsetup status --target cache` is the full command that telegraf will run for debugging purposes. ## Configuration ``` [[inputs.dmcache]] -## Whether to report per-device stats or not - per_device = true + ## Whether to report per-device stats or not + per_device = true ``` diff --git a/plugins/inputs/dmcache/dmcache.go b/plugins/inputs/dmcache/dmcache.go index bfcb6dadcb751..c36eed636af77 100644 --- a/plugins/inputs/dmcache/dmcache.go +++ b/plugins/inputs/dmcache/dmcache.go @@ -14,8 +14,8 @@ type DMCache struct { } var sampleConfig = ` - ## Whether to report per-device stats or not - per_device = true + ## Whether to report per-device stats or not + per_device = true ` func (c *DMCache) SampleConfig() string { diff --git a/plugins/inputs/dmcache/dmcache_linux.go b/plugins/inputs/dmcache/dmcache_linux.go index ed8b88c6809d3..b64b19056a4fd 100644 --- a/plugins/inputs/dmcache/dmcache_linux.go +++ b/plugins/inputs/dmcache/dmcache_linux.go @@ -6,6 +6,8 @@ import ( "strconv" "strings" + "errors" + "github.com/influxdata/telegraf" ) @@ -31,10 +33,7 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { return err } - var total map[string]interface{} - if !c.PerDevice { - total = make(map[string]interface{}) - } + total := make(map[string]interface{}) for _, s := range outputLines { fields := make(map[string]interface{}) @@ -50,58 +49,86 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { if c.PerDevice { tags := map[string]string{"device": data["device"].(string)} acc.AddFields(metricName, fields, tags) - } else { - aggregateStats(total, fields) } + aggregateStats(total, fields) } - if !c.PerDevice { - acc.AddFields(metricName, total, nil) - } + acc.AddFields(metricName, total, map[string]string{"device": "all"}) return nil } -func parseDMSetupStatus(line string) (status map[string]interface{}, err error) { - defer func() { - if r := recover(); r != nil { - status = nil - err = r.(error) - } - }() - - values := strings.Split(line, " ") - status = make(map[string]interface{}) +func parseDMSetupStatus(line string) (map[string]interface{}, error) { + var err error + status := make(map[string]interface{}) + values := strings.Fields(line) + if len(values) < 15 { + return nil, errors.New("dmsetup status data have invalid format") + } status["device"] = values[0][:len(values[0])-1] - status["length"] = toInt(values[2]) + status["length"], err = strconv.Atoi(values[2]) + if err != nil { + return nil, err + } status["target"] = values[3] - status["metadata_blocksize"] = toInt(values[4]) - status["metadata_used"] = toInt(strings.Split(values[5], "/")[0]) - status["metadata_total"] = toInt(strings.Split(values[5], "/")[1]) - status["cache_blocksize"] = toInt(values[6]) - status["cache_used"] = toInt(strings.Split(values[7], "/")[0]) - status["cache_total"] = toInt(strings.Split(values[7], "/")[1]) - status["read_hits"] = toInt(values[8]) - status["read_misses"] = toInt(values[9]) - status["write_hits"] = toInt(values[10]) - status["write_misses"] = toInt(values[11]) - status["demotions"] = toInt(values[12]) - status["promotions"] = toInt(values[13]) - status["dirty"] = toInt(values[14]) + status["metadata_blocksize"], err = strconv.Atoi(values[4]) + if err != nil { + return nil, err + } + status["metadata_used"], err = strconv.Atoi(strings.Split(values[5], "/")[0]) + if err != nil { + return nil, err + } + status["metadata_total"], err = strconv.Atoi(strings.Split(values[5], "/")[1]) + if err != nil { + return nil, err + } + status["cache_blocksize"], err = strconv.Atoi(values[6]) + if err != nil { + return nil, err + } + status["cache_used"], err = strconv.Atoi(strings.Split(values[7], "/")[0]) + if err != nil { + return nil, err + } + status["cache_total"], err = strconv.Atoi(strings.Split(values[7], "/")[1]) + if err != nil { + return nil, err + } + status["read_hits"], err = strconv.Atoi(values[8]) + if err != nil { + return nil, err + } + status["read_misses"], err = strconv.Atoi(values[9]) + if err != nil { + return nil, err + } + status["write_hits"], err = strconv.Atoi(values[10]) + if err != nil { + return nil, err + } + status["write_misses"], err = strconv.Atoi(values[11]) + if err != nil { + return nil, err + } + status["demotions"], err = strconv.Atoi(values[12]) + if err != nil { + return nil, err + } + status["promotions"], err = strconv.Atoi(values[13]) + if err != nil { + return nil, err + } + status["dirty"], err = strconv.Atoi(values[14]) + if err != nil { + return nil, err + } status["blocksize"] = 512 return status, nil } -func toInt(s string) int { - i, err := strconv.Atoi(s) - if err != nil { - panic(err) - } - return i -} - func calculateSize(data map[string]interface{}, key string) (value int) { if key == "metadata_free" { value = data["metadata_total"].(int) - data["metadata_used"].(int) diff --git a/plugins/inputs/dmcache/dmcache_test.go b/plugins/inputs/dmcache/dmcache_test.go index 42d5072990802..d4295b38b268b 100644 --- a/plugins/inputs/dmcache/dmcache_test.go +++ b/plugins/inputs/dmcache/dmcache_test.go @@ -1,5 +1,3 @@ -// +build linux - package dmcache import ( @@ -62,6 +60,25 @@ func TestDMCacheStats_1(t *testing.T) { "dirty": 0, } acc.AssertContainsTaggedFields(t, "dmcache", fields2, tags2) + + tags3 := map[string]string{ + "device": "all", + } + + fields3 := map[string]interface{}{ + "metadata_used": 300523520, + "metadata_free": 11216781312, + "cache_used": 3538944, + "cache_free": 1716188741632, + "read_hits": 194314240, + "read_misses": 92461989888, + "write_hits": 21299200, + "write_misses": 34397618176, + "demotions": 0, + "promotions": 1835008, + "dirty": 0, + } + acc.AssertContainsTaggedFields(t, "dmcache", fields3, tags3) } var dmc2 = &DMCache{ @@ -75,7 +92,9 @@ func TestDMCacheStats_2(t *testing.T) { err := dmc2.Gather(&acc) require.NoError(t, err) - tags := map[string]string{} + tags := map[string]string{ + "device": "all", + } fields := map[string]interface{}{ "metadata_used": 300523520, From f5e762f17115c2ec5f8fc943dd915ccfb8f87e22 Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Mon, 3 Apr 2017 16:02:59 +0300 Subject: [PATCH 03/18] refactoring and new tests --- plugins/inputs/dmcache/dmcache.go | 36 ++++++++------- plugins/inputs/dmcache/dmcache_linux.go | 23 +++++++--- plugins/inputs/dmcache/dmcache_test.go | 59 +++++++++++++++++++++++-- 3 files changed, 90 insertions(+), 28 deletions(-) diff --git a/plugins/inputs/dmcache/dmcache.go b/plugins/inputs/dmcache/dmcache.go index c36eed636af77..b933cc098b10a 100644 --- a/plugins/inputs/dmcache/dmcache.go +++ b/plugins/inputs/dmcache/dmcache.go @@ -9,8 +9,8 @@ import ( ) type DMCache struct { - PerDevice bool `toml:"per_device"` - rawStatus func() ([]string, error) + PerDevice bool `toml:"per_device"` + getCurrentStatus func() ([]string, error) } var sampleConfig = ` @@ -26,24 +26,26 @@ func (c *DMCache) Description() string { return "Provide a native collection for dmsetup based statistics for dm-cache" } +func dmSetupStatus() ([]string, error) { + out, err := exec.Command("/bin/sh", "-c", "sudo /sbin/dmsetup status --target cache").Output() + if err != nil { + return nil, err + } + if string(out) == "No devices found\n" { + return []string{}, nil + } + + outString := strings.TrimRight(string(out), "\n") + status := strings.Split(outString, "\n") + + return status, nil +} + func init() { inputs.Add("dmcache", func() telegraf.Input { return &DMCache{ - PerDevice: true, - rawStatus: func() ([]string, error) { - out, err := exec.Command("/bin/sh", "-c", "sudo /sbin/dmsetup status --target cache").Output() - if err != nil { - return nil, err - } - if string(out) == "No devices found\n" { - return nil, nil - } - - status := strings.Split(string(out), "\n") - status = status[:len(status)-1] // removing last empty line - - return status, nil - }, + PerDevice: true, + getCurrentStatus: dmSetupStatus, } }) } diff --git a/plugins/inputs/dmcache/dmcache_linux.go b/plugins/inputs/dmcache/dmcache_linux.go index b64b19056a4fd..12fc2a0919401 100644 --- a/plugins/inputs/dmcache/dmcache_linux.go +++ b/plugins/inputs/dmcache/dmcache_linux.go @@ -28,7 +28,7 @@ var fieldNames = [...]string{ } func (c *DMCache) Gather(acc telegraf.Accumulator) error { - outputLines, err := c.rawStatus() + outputLines, err := c.getCurrentStatus() if err != nil { return err } @@ -60,13 +60,14 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { func parseDMSetupStatus(line string) (map[string]interface{}, error) { var err error + parseError := errors.New("Output from dmsetup could not be parsed") status := make(map[string]interface{}) values := strings.Fields(line) if len(values) < 15 { - return nil, errors.New("dmsetup status data have invalid format") + return nil, parseError } - status["device"] = values[0][:len(values[0])-1] + status["device"] = strings.TrimRight(values[0], ":") status["length"], err = strconv.Atoi(values[2]) if err != nil { return nil, err @@ -76,11 +77,15 @@ func parseDMSetupStatus(line string) (map[string]interface{}, error) { if err != nil { return nil, err } - status["metadata_used"], err = strconv.Atoi(strings.Split(values[5], "/")[0]) + metadata := strings.Split(values[5], "/") + if len(metadata) != 2 { + return nil, parseError + } + status["metadata_used"], err = strconv.Atoi(metadata[0]) if err != nil { return nil, err } - status["metadata_total"], err = strconv.Atoi(strings.Split(values[5], "/")[1]) + status["metadata_total"], err = strconv.Atoi(metadata[1]) if err != nil { return nil, err } @@ -88,11 +93,15 @@ func parseDMSetupStatus(line string) (map[string]interface{}, error) { if err != nil { return nil, err } - status["cache_used"], err = strconv.Atoi(strings.Split(values[7], "/")[0]) + cache := strings.Split(values[7], "/") + if len(cache) != 2 { + return nil, parseError + } + status["cache_used"], err = strconv.Atoi(cache[0]) if err != nil { return nil, err } - status["cache_total"], err = strconv.Atoi(strings.Split(values[7], "/")[1]) + status["cache_total"], err = strconv.Atoi(cache[1]) if err != nil { return nil, err } diff --git a/plugins/inputs/dmcache/dmcache_test.go b/plugins/inputs/dmcache/dmcache_test.go index d4295b38b268b..b35dc0eede9f3 100644 --- a/plugins/inputs/dmcache/dmcache_test.go +++ b/plugins/inputs/dmcache/dmcache_test.go @@ -1,6 +1,7 @@ package dmcache import ( + "errors" "testing" "github.com/influxdata/telegraf/testutil" @@ -15,8 +16,8 @@ func output2Devices() ([]string, error) { } var dmc1 = &DMCache{ - PerDevice: true, - rawStatus: output2Devices, + PerDevice: true, + getCurrentStatus: output2Devices, } func TestDMCacheStats_1(t *testing.T) { @@ -82,8 +83,8 @@ func TestDMCacheStats_1(t *testing.T) { } var dmc2 = &DMCache{ - PerDevice: false, - rawStatus: output2Devices, + PerDevice: false, + getCurrentStatus: output2Devices, } func TestDMCacheStats_2(t *testing.T) { @@ -111,3 +112,53 @@ func TestDMCacheStats_2(t *testing.T) { } acc.AssertContainsTaggedFields(t, "dmcache", fields, tags) } + +func outputNoDevices() ([]string, error) { + return []string{}, nil +} + +var dmc3 = &DMCache{ + PerDevice: true, + getCurrentStatus: outputNoDevices, +} + +func TestDMCacheStats_3(t *testing.T) { + var acc testutil.Accumulator + + err := dmc3.Gather(&acc) + require.NoError(t, err) +} + +func noDMSetup() ([]string, error) { + return []string{}, errors.New("dmsetup doesn't exist") +} + +var dmc4 = &DMCache{ + PerDevice: true, + getCurrentStatus: noDMSetup, +} + +func TestDMCacheStats_4(t *testing.T) { + var acc testutil.Accumulator + + err := dmc4.Gather(&acc) + require.Error(t, err) +} + +func badFormat() ([]string, error) { + return []string{ + "cs-1: 0 4883791872 cache 8 1018/1501122 512 7/464962 139 352643 ", + }, nil +} + +var dmc5 = &DMCache{ + PerDevice: true, + getCurrentStatus: badFormat, +} + +func TestDMCacheStats_5(t *testing.T) { + var acc testutil.Accumulator + + err := dmc5.Gather(&acc) + require.Error(t, err) +} From e5943bbc1f89a1cc4515e15222df3d99ece29aee Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Tue, 4 Apr 2017 17:40:52 +0300 Subject: [PATCH 04/18] removed synthesized values --- plugins/inputs/dmcache/dmcache_linux.go | 52 ++---------- plugins/inputs/dmcache/dmcache_test.go | 104 ++++++++++++++---------- 2 files changed, 68 insertions(+), 88 deletions(-) diff --git a/plugins/inputs/dmcache/dmcache_linux.go b/plugins/inputs/dmcache/dmcache_linux.go index 12fc2a0919401..84cece5684ab7 100644 --- a/plugins/inputs/dmcache/dmcache_linux.go +++ b/plugins/inputs/dmcache/dmcache_linux.go @@ -13,20 +13,6 @@ import ( const metricName = "dmcache" -var fieldNames = [...]string{ - "metadata_used", - "metadata_free", - "cache_used", - "cache_free", - "read_hits", - "read_misses", - "write_hits", - "write_misses", - "demotions", - "promotions", - "dirty", -} - func (c *DMCache) Gather(acc telegraf.Accumulator) error { outputLines, err := c.getCurrentStatus() if err != nil { @@ -36,18 +22,13 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { total := make(map[string]interface{}) for _, s := range outputLines { - fields := make(map[string]interface{}) - data, err := parseDMSetupStatus(s) + fields, err := parseDMSetupStatus(s) if err != nil { return err } - for _, f := range fieldNames { - fields[f] = calculateSize(data, f) - } - if c.PerDevice { - tags := map[string]string{"device": data["device"].(string)} + tags := map[string]string{"device": fields["device"].(string)} acc.AddFields(metricName, fields, tags) } aggregateStats(total, fields) @@ -133,35 +114,18 @@ func parseDMSetupStatus(line string) (map[string]interface{}, error) { if err != nil { return nil, err } - status["blocksize"] = 512 return status, nil } -func calculateSize(data map[string]interface{}, key string) (value int) { - if key == "metadata_free" { - value = data["metadata_total"].(int) - data["metadata_used"].(int) - } else if key == "cache_free" { - value = data["cache_total"].(int) - data["cache_used"].(int) - data["dirty"].(int) - } else { - value = data[key].(int) - } - - if key == "metadata_free" || key == "metadata_used" { - value = value * data["blocksize"].(int) * data["metadata_blocksize"].(int) - } else { - value = value * data["blocksize"].(int) * data["cache_blocksize"].(int) - } - - return -} - func aggregateStats(total, fields map[string]interface{}) { for key, value := range fields { - if _, ok := total[key]; ok { - total[key] = total[key].(int) + value.(int) - } else { - total[key] = value.(int) + if _, ok := value.(int); ok { + if _, ok := total[key]; ok { + total[key] = total[key].(int) + value.(int) + } else { + total[key] = value.(int) + } } } } diff --git a/plugins/inputs/dmcache/dmcache_test.go b/plugins/inputs/dmcache/dmcache_test.go index b35dc0eede9f3..2c317f9da16b7 100644 --- a/plugins/inputs/dmcache/dmcache_test.go +++ b/plugins/inputs/dmcache/dmcache_test.go @@ -30,17 +30,22 @@ func TestDMCacheStats_1(t *testing.T) { "device": "cs-1", } fields1 := map[string]interface{}{ - "metadata_used": 4169728, - "metadata_free": 6144425984, - "cache_used": 1835008, - "cache_free": 121885163520, - "read_hits": 36438016, - "read_misses": 92443246592, - "write_hits": 3932160, - "write_misses": 12058624, - "demotions": 0, - "promotions": 1835008, - "dirty": 0, + "device": "cs-1", + "length": 4883791872, + "target": "cache", + "metadata_blocksize": 8, + "metadata_used": 1018, + "metadata_total": 1501122, + "cache_blocksize": 512, + "cache_used": 7, + "cache_total": 464962, + "read_hits": 139, + "read_misses": 352643, + "write_hits": 15, + "write_misses": 46, + "demotions": 0, + "promotions": 7, + "dirty": 0, } acc.AssertContainsTaggedFields(t, "dmcache", fields1, tags1) @@ -48,17 +53,22 @@ func TestDMCacheStats_1(t *testing.T) { "device": "cs-2", } fields2 := map[string]interface{}{ - "metadata_used": 296353792, - "metadata_free": 5072355328, - "cache_used": 1703936, - "cache_free": 1594303578112, - "read_hits": 157876224, - "read_misses": 18743296, - "write_hits": 17367040, - "write_misses": 34385559552, - "demotions": 0, - "promotions": 0, - "dirty": 0, + "device": "cs-2", + "length": 4294967296, + "target": "cache", + "metadata_blocksize": 8, + "metadata_used": 72352, + "metadata_total": 1310720, + "cache_blocksize": 128, + "cache_used": 26, + "cache_total": 24327168, + "read_hits": 2409, + "read_misses": 286, + "write_hits": 265, + "write_misses": 524682, + "demotions": 0, + "promotions": 0, + "dirty": 0, } acc.AssertContainsTaggedFields(t, "dmcache", fields2, tags2) @@ -67,17 +77,20 @@ func TestDMCacheStats_1(t *testing.T) { } fields3 := map[string]interface{}{ - "metadata_used": 300523520, - "metadata_free": 11216781312, - "cache_used": 3538944, - "cache_free": 1716188741632, - "read_hits": 194314240, - "read_misses": 92461989888, - "write_hits": 21299200, - "write_misses": 34397618176, - "demotions": 0, - "promotions": 1835008, - "dirty": 0, + "length": 9178759168, + "metadata_blocksize": 16, + "metadata_used": 73370, + "metadata_total": 2811842, + "cache_blocksize": 640, + "cache_used": 33, + "cache_total": 24792130, + "read_hits": 2548, + "read_misses": 352929, + "write_hits": 280, + "write_misses": 524728, + "demotions": 0, + "promotions": 7, + "dirty": 0, } acc.AssertContainsTaggedFields(t, "dmcache", fields3, tags3) } @@ -98,17 +111,20 @@ func TestDMCacheStats_2(t *testing.T) { } fields := map[string]interface{}{ - "metadata_used": 300523520, - "metadata_free": 11216781312, - "cache_used": 3538944, - "cache_free": 1716188741632, - "read_hits": 194314240, - "read_misses": 92461989888, - "write_hits": 21299200, - "write_misses": 34397618176, - "demotions": 0, - "promotions": 1835008, - "dirty": 0, + "length": 9178759168, + "metadata_blocksize": 16, + "metadata_used": 73370, + "metadata_total": 2811842, + "cache_blocksize": 640, + "cache_used": 33, + "cache_total": 24792130, + "read_hits": 2548, + "read_misses": 352929, + "write_hits": 280, + "write_misses": 524728, + "demotions": 0, + "promotions": 7, + "dirty": 0, } acc.AssertContainsTaggedFields(t, "dmcache", fields, tags) } From 5eadca576cb97190b946d6a4a42b6ed69aa44472 Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Wed, 5 Apr 2017 11:25:19 +0300 Subject: [PATCH 05/18] updated readme --- plugins/inputs/dmcache/README.md | 36 ++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/dmcache/README.md b/plugins/inputs/dmcache/README.md index ae4142e9aa2fa..8e45726860dc2 100644 --- a/plugins/inputs/dmcache/README.md +++ b/plugins/inputs/dmcache/README.md @@ -6,10 +6,42 @@ This plugin requires sudo, that is why you should setup and be sure that the tel `sudo /sbin/dmsetup status --target cache` is the full command that telegraf will run for debugging purposes. -## Configuration +### Configuration -``` +```toml [[inputs.dmcache]] ## Whether to report per-device stats or not per_device = true ``` + +### Measurements & Fields: + +- dmcache + - length + - target + - metadata_blocksize + - metadata_used + - metadata_total + - cache_blocksize + - cache_used + - cache_total + - read_hits + - read_misses + - write_hits + - write_misses + - demotions + - promotions + - dirty + +### Tags: + +- All measurements have the following tags: + - device + +### Example Output: + +``` +$ ./telegraf --test --config /etc/telegraf/telegraf.conf --input-filter dmcache +* Plugin: inputs.dmcache, Collection 1 +> dmcache,bu=linux,cls=server,dc=colo,device=vg02-splunk_data_lv,env=production,host=hostname,sr=splunk_indexer,trd=false cache_free=0i,cache_used=1099511627776i,demotions=3265223720960i,dirty=0i,metadata_free=1060880384i,metadata_used=12861440i,promotions=3265223720960i,read_hits=995134034411520i,read_misses=916807089127424i,write_hits=195107267543040i,write_misses=563725346013184i 1491362011000000000 +``` \ No newline at end of file From dc4c41943d96bb8f3078210a2b7a1c080a7fea54 Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Wed, 5 Apr 2017 12:08:41 +0300 Subject: [PATCH 06/18] refactored map to struct --- plugins/inputs/dmcache/dmcache.go | 18 ---- plugins/inputs/dmcache/dmcache_linux.go | 135 ++++++++++++++++-------- plugins/inputs/dmcache/dmcache_test.go | 4 - 3 files changed, 92 insertions(+), 65 deletions(-) diff --git a/plugins/inputs/dmcache/dmcache.go b/plugins/inputs/dmcache/dmcache.go index b933cc098b10a..25a398194edf8 100644 --- a/plugins/inputs/dmcache/dmcache.go +++ b/plugins/inputs/dmcache/dmcache.go @@ -1,9 +1,6 @@ package dmcache import ( - "os/exec" - "strings" - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" ) @@ -26,21 +23,6 @@ func (c *DMCache) Description() string { return "Provide a native collection for dmsetup based statistics for dm-cache" } -func dmSetupStatus() ([]string, error) { - out, err := exec.Command("/bin/sh", "-c", "sudo /sbin/dmsetup status --target cache").Output() - if err != nil { - return nil, err - } - if string(out) == "No devices found\n" { - return []string{}, nil - } - - outString := strings.TrimRight(string(out), "\n") - status := strings.Split(outString, "\n") - - return status, nil -} - func init() { inputs.Add("dmcache", func() telegraf.Input { return &DMCache{ diff --git a/plugins/inputs/dmcache/dmcache_linux.go b/plugins/inputs/dmcache/dmcache_linux.go index 84cece5684ab7..8c698834b384f 100644 --- a/plugins/inputs/dmcache/dmcache_linux.go +++ b/plugins/inputs/dmcache/dmcache_linux.go @@ -3,6 +3,7 @@ package dmcache import ( + "os/exec" "strconv" "strings" @@ -13,6 +14,25 @@ import ( const metricName = "dmcache" +type cacheStatus struct { + device string + length int + target string + metadataBlocksize int + metadataUsed int + metadataTotal int + cacheBlocksize int + cacheUsed int + cacheTotal int + readHits int + readMisses int + writeHits int + writeMisses int + demotions int + promotions int + dirty int +} + func (c *DMCache) Gather(acc telegraf.Accumulator) error { outputLines, err := c.getCurrentStatus() if err != nil { @@ -22,13 +42,29 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { total := make(map[string]interface{}) for _, s := range outputLines { - fields, err := parseDMSetupStatus(s) + status, err := parseDMSetupStatus(s) if err != nil { return err } + fields := make(map[string]interface{}) + fields["length"] = status.length + fields["metadata_blocksize"] = status.metadataBlocksize + fields["metadata_used"] = status.metadataUsed + fields["metadata_total"] = status.metadataTotal + fields["cache_blocksize"] = status.cacheBlocksize + fields["cache_used"] = status.cacheUsed + fields["cache_total"] = status.cacheTotal + fields["read_hits"] = status.readHits + fields["read_misses"] = status.readMisses + fields["write_hits"] = status.writeHits + fields["write_misses"] = status.writeMisses + fields["demotions"] = status.demotions + fields["promotions"] = status.promotions + fields["dirty"] = status.dirty + if c.PerDevice { - tags := map[string]string{"device": fields["device"].(string)} + tags := map[string]string{"device": status.device} acc.AddFields(metricName, fields, tags) } aggregateStats(total, fields) @@ -39,80 +75,80 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { return nil } -func parseDMSetupStatus(line string) (map[string]interface{}, error) { +func parseDMSetupStatus(line string) (cacheStatus, error) { var err error parseError := errors.New("Output from dmsetup could not be parsed") - status := make(map[string]interface{}) + status := cacheStatus{} values := strings.Fields(line) if len(values) < 15 { - return nil, parseError + return cacheStatus{}, parseError } - status["device"] = strings.TrimRight(values[0], ":") - status["length"], err = strconv.Atoi(values[2]) + status.device = strings.TrimRight(values[0], ":") + status.length, err = strconv.Atoi(values[2]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["target"] = values[3] - status["metadata_blocksize"], err = strconv.Atoi(values[4]) + status.target = values[3] + status.metadataBlocksize, err = strconv.Atoi(values[4]) if err != nil { - return nil, err + return cacheStatus{}, err } metadata := strings.Split(values[5], "/") if len(metadata) != 2 { - return nil, parseError + return cacheStatus{}, parseError } - status["metadata_used"], err = strconv.Atoi(metadata[0]) + status.metadataUsed, err = strconv.Atoi(metadata[0]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["metadata_total"], err = strconv.Atoi(metadata[1]) + status.metadataTotal, err = strconv.Atoi(metadata[1]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["cache_blocksize"], err = strconv.Atoi(values[6]) + status.cacheBlocksize, err = strconv.Atoi(values[6]) if err != nil { - return nil, err + return cacheStatus{}, err } cache := strings.Split(values[7], "/") if len(cache) != 2 { - return nil, parseError + return cacheStatus{}, parseError } - status["cache_used"], err = strconv.Atoi(cache[0]) + status.cacheUsed, err = strconv.Atoi(cache[0]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["cache_total"], err = strconv.Atoi(cache[1]) + status.cacheTotal, err = strconv.Atoi(cache[1]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["read_hits"], err = strconv.Atoi(values[8]) + status.readHits, err = strconv.Atoi(values[8]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["read_misses"], err = strconv.Atoi(values[9]) + status.readMisses, err = strconv.Atoi(values[9]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["write_hits"], err = strconv.Atoi(values[10]) + status.writeHits, err = strconv.Atoi(values[10]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["write_misses"], err = strconv.Atoi(values[11]) + status.writeMisses, err = strconv.Atoi(values[11]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["demotions"], err = strconv.Atoi(values[12]) + status.demotions, err = strconv.Atoi(values[12]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["promotions"], err = strconv.Atoi(values[13]) + status.promotions, err = strconv.Atoi(values[13]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["dirty"], err = strconv.Atoi(values[14]) + status.dirty, err = strconv.Atoi(values[14]) if err != nil { - return nil, err + return cacheStatus{}, err } return status, nil @@ -120,12 +156,25 @@ func parseDMSetupStatus(line string) (map[string]interface{}, error) { func aggregateStats(total, fields map[string]interface{}) { for key, value := range fields { - if _, ok := value.(int); ok { - if _, ok := total[key]; ok { - total[key] = total[key].(int) + value.(int) - } else { - total[key] = value.(int) - } + if _, ok := total[key]; ok { + total[key] = total[key].(int) + value.(int) + } else { + total[key] = value.(int) } } } + +func dmSetupStatus() ([]string, error) { + out, err := exec.Command("/bin/sh", "-c", "sudo /sbin/dmsetup status --target cache").Output() + if err != nil { + return nil, err + } + if string(out) == "No devices found\n" { + return []string{}, nil + } + + outString := strings.TrimRight(string(out), "\n") + status := strings.Split(outString, "\n") + + return status, nil +} diff --git a/plugins/inputs/dmcache/dmcache_test.go b/plugins/inputs/dmcache/dmcache_test.go index 2c317f9da16b7..e30b91e2ee6f4 100644 --- a/plugins/inputs/dmcache/dmcache_test.go +++ b/plugins/inputs/dmcache/dmcache_test.go @@ -30,9 +30,7 @@ func TestDMCacheStats_1(t *testing.T) { "device": "cs-1", } fields1 := map[string]interface{}{ - "device": "cs-1", "length": 4883791872, - "target": "cache", "metadata_blocksize": 8, "metadata_used": 1018, "metadata_total": 1501122, @@ -53,9 +51,7 @@ func TestDMCacheStats_1(t *testing.T) { "device": "cs-2", } fields2 := map[string]interface{}{ - "device": "cs-2", "length": 4294967296, - "target": "cache", "metadata_blocksize": 8, "metadata_used": 72352, "metadata_total": 1310720, From b8d6d960c66f423cc833b1d46ebc7b13dabfff67 Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Thu, 6 Apr 2017 15:54:26 +0300 Subject: [PATCH 07/18] refactoring: switching to fields at the last moment --- plugins/inputs/dmcache/README.md | 4 +- plugins/inputs/dmcache/dmcache_linux.go | 66 ++++++++++++++----------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/plugins/inputs/dmcache/README.md b/plugins/inputs/dmcache/README.md index 8e45726860dc2..536d3f518bcaa 100644 --- a/plugins/inputs/dmcache/README.md +++ b/plugins/inputs/dmcache/README.md @@ -43,5 +43,5 @@ This plugin requires sudo, that is why you should setup and be sure that the tel ``` $ ./telegraf --test --config /etc/telegraf/telegraf.conf --input-filter dmcache * Plugin: inputs.dmcache, Collection 1 -> dmcache,bu=linux,cls=server,dc=colo,device=vg02-splunk_data_lv,env=production,host=hostname,sr=splunk_indexer,trd=false cache_free=0i,cache_used=1099511627776i,demotions=3265223720960i,dirty=0i,metadata_free=1060880384i,metadata_used=12861440i,promotions=3265223720960i,read_hits=995134034411520i,read_misses=916807089127424i,write_hits=195107267543040i,write_misses=563725346013184i 1491362011000000000 -``` \ No newline at end of file +> dmcache,device=example cache_blocksize=0i,read_hits=995134034411520i,read_misses=916807089127424i,write_hits=195107267543040i,metadata_used=12861440i,write_misses=563725346013184i,promotions=3265223720960i,dirty=0i,metadata_blocksize=0i,cache_used=1099511627776ii,cache_total=0i,length=0i,metadata_total=1073741824i,demotions=3265223720960i 1491482035000000000 +``` diff --git a/plugins/inputs/dmcache/dmcache_linux.go b/plugins/inputs/dmcache/dmcache_linux.go index 8c698834b384f..7ac1c96cae0f1 100644 --- a/plugins/inputs/dmcache/dmcache_linux.go +++ b/plugins/inputs/dmcache/dmcache_linux.go @@ -39,7 +39,7 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { return err } - total := make(map[string]interface{}) + totalStatus := cacheStatus{} for _, s := range outputLines { status, err := parseDMSetupStatus(s) @@ -47,30 +47,14 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { return err } - fields := make(map[string]interface{}) - fields["length"] = status.length - fields["metadata_blocksize"] = status.metadataBlocksize - fields["metadata_used"] = status.metadataUsed - fields["metadata_total"] = status.metadataTotal - fields["cache_blocksize"] = status.cacheBlocksize - fields["cache_used"] = status.cacheUsed - fields["cache_total"] = status.cacheTotal - fields["read_hits"] = status.readHits - fields["read_misses"] = status.readMisses - fields["write_hits"] = status.writeHits - fields["write_misses"] = status.writeMisses - fields["demotions"] = status.demotions - fields["promotions"] = status.promotions - fields["dirty"] = status.dirty - if c.PerDevice { tags := map[string]string{"device": status.device} - acc.AddFields(metricName, fields, tags) + acc.AddFields(metricName, toFields(status), tags) } - aggregateStats(total, fields) + aggregateStats(&totalStatus, status) } - acc.AddFields(metricName, total, map[string]string{"device": "all"}) + acc.AddFields(metricName, toFields(totalStatus), map[string]string{"device": "all"}) return nil } @@ -154,14 +138,40 @@ func parseDMSetupStatus(line string) (cacheStatus, error) { return status, nil } -func aggregateStats(total, fields map[string]interface{}) { - for key, value := range fields { - if _, ok := total[key]; ok { - total[key] = total[key].(int) + value.(int) - } else { - total[key] = value.(int) - } - } +func aggregateStats(totalStatus *cacheStatus, status cacheStatus) { + totalStatus.length += status.length + totalStatus.metadataBlocksize += status.metadataBlocksize + totalStatus.metadataUsed += status.metadataUsed + totalStatus.metadataTotal += status.metadataTotal + totalStatus.cacheBlocksize += status.cacheBlocksize + totalStatus.cacheUsed += status.cacheUsed + totalStatus.cacheTotal += status.cacheTotal + totalStatus.readHits += status.readHits + totalStatus.readMisses += status.readMisses + totalStatus.writeHits += status.writeHits + totalStatus.writeMisses += status.writeMisses + totalStatus.demotions += status.demotions + totalStatus.promotions += status.promotions + totalStatus.dirty += status.dirty +} + +func toFields(status cacheStatus) map[string]interface{} { + fields := make(map[string]interface{}) + fields["length"] = status.length + fields["metadata_blocksize"] = status.metadataBlocksize + fields["metadata_used"] = status.metadataUsed + fields["metadata_total"] = status.metadataTotal + fields["cache_blocksize"] = status.cacheBlocksize + fields["cache_used"] = status.cacheUsed + fields["cache_total"] = status.cacheTotal + fields["read_hits"] = status.readHits + fields["read_misses"] = status.readMisses + fields["write_hits"] = status.writeHits + fields["write_misses"] = status.writeMisses + fields["demotions"] = status.demotions + fields["promotions"] = status.promotions + fields["dirty"] = status.dirty + return fields } func dmSetupStatus() ([]string, error) { From 039d55853e459249b0de3975649f70a1abd77a9e Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Fri, 7 Apr 2017 11:49:27 +0300 Subject: [PATCH 08/18] tests refactoring --- plugins/inputs/dmcache/dmcache_test.go | 107 ++++++++++++------------- 1 file changed, 50 insertions(+), 57 deletions(-) diff --git a/plugins/inputs/dmcache/dmcache_test.go b/plugins/inputs/dmcache/dmcache_test.go index e30b91e2ee6f4..c5989c413d9c0 100644 --- a/plugins/inputs/dmcache/dmcache_test.go +++ b/plugins/inputs/dmcache/dmcache_test.go @@ -8,22 +8,25 @@ import ( "github.com/stretchr/testify/require" ) -func output2Devices() ([]string, error) { - return []string{ +var ( + measurement = "dmcache" + badFormatOutput = []string{"cs-1: 0 4883791872 cache 8 1018/1501122 512 7/464962 139 352643 "} + good2DevicesFormatOutput = []string{ "cs-1: 0 4883791872 cache 8 1018/1501122 512 7/464962 139 352643 15 46 0 7 0 1 writeback 2 migration_threshold 2048 mq 10 random_threshold 4 sequential_threshold 512 discard_promote_adjustment 1 read_promote_adjustment 4 write_promote_adjustment 8", "cs-2: 0 4294967296 cache 8 72352/1310720 128 26/24327168 2409 286 265 524682 0 0 0 1 writethrough 2 migration_threshold 2048 mq 10 random_threshold 4 sequential_threshold 512 discard_promote_adjustment 1 read_promote_adjustment 4 write_promote_adjustment 8", - }, nil -} - -var dmc1 = &DMCache{ - PerDevice: true, - getCurrentStatus: output2Devices, -} + } +) -func TestDMCacheStats_1(t *testing.T) { +func TestPerDeviceGoodOutput(t *testing.T) { var acc testutil.Accumulator + var plugin = &DMCache{ + PerDevice: true, + getCurrentStatus: func() ([]string, error) { + return good2DevicesFormatOutput, nil + }, + } - err := dmc1.Gather(&acc) + err := plugin.Gather(&acc) require.NoError(t, err) tags1 := map[string]string{ @@ -45,7 +48,7 @@ func TestDMCacheStats_1(t *testing.T) { "promotions": 7, "dirty": 0, } - acc.AssertContainsTaggedFields(t, "dmcache", fields1, tags1) + acc.AssertContainsTaggedFields(t, measurement, fields1, tags1) tags2 := map[string]string{ "device": "cs-2", @@ -66,7 +69,7 @@ func TestDMCacheStats_1(t *testing.T) { "promotions": 0, "dirty": 0, } - acc.AssertContainsTaggedFields(t, "dmcache", fields2, tags2) + acc.AssertContainsTaggedFields(t, measurement, fields2, tags2) tags3 := map[string]string{ "device": "all", @@ -88,18 +91,19 @@ func TestDMCacheStats_1(t *testing.T) { "promotions": 7, "dirty": 0, } - acc.AssertContainsTaggedFields(t, "dmcache", fields3, tags3) + acc.AssertContainsTaggedFields(t, measurement, fields3, tags3) } -var dmc2 = &DMCache{ - PerDevice: false, - getCurrentStatus: output2Devices, -} - -func TestDMCacheStats_2(t *testing.T) { +func TestNotPerDeviceGoodOutput(t *testing.T) { var acc testutil.Accumulator + var plugin = &DMCache{ + PerDevice: false, + getCurrentStatus: func() ([]string, error) { + return good2DevicesFormatOutput, nil + }, + } - err := dmc2.Gather(&acc) + err := plugin.Gather(&acc) require.NoError(t, err) tags := map[string]string{ @@ -122,55 +126,44 @@ func TestDMCacheStats_2(t *testing.T) { "promotions": 7, "dirty": 0, } - acc.AssertContainsTaggedFields(t, "dmcache", fields, tags) -} - -func outputNoDevices() ([]string, error) { - return []string{}, nil -} - -var dmc3 = &DMCache{ - PerDevice: true, - getCurrentStatus: outputNoDevices, + acc.AssertContainsTaggedFields(t, measurement, fields, tags) } -func TestDMCacheStats_3(t *testing.T) { +func TestNoDevicesOutput(t *testing.T) { var acc testutil.Accumulator + var plugin = &DMCache{ + PerDevice: true, + getCurrentStatus: func() ([]string, error) { + return []string{}, nil + }, + } - err := dmc3.Gather(&acc) + err := plugin.Gather(&acc) require.NoError(t, err) } -func noDMSetup() ([]string, error) { - return []string{}, errors.New("dmsetup doesn't exist") -} - -var dmc4 = &DMCache{ - PerDevice: true, - getCurrentStatus: noDMSetup, -} - -func TestDMCacheStats_4(t *testing.T) { +func TestErrorDuringGettingStatus(t *testing.T) { var acc testutil.Accumulator + var plugin = &DMCache{ + PerDevice: true, + getCurrentStatus: func() ([]string, error) { + return nil, errors.New("dmsetup doesn't exist") + }, + } - err := dmc4.Gather(&acc) + err := plugin.Gather(&acc) require.Error(t, err) } -func badFormat() ([]string, error) { - return []string{ - "cs-1: 0 4883791872 cache 8 1018/1501122 512 7/464962 139 352643 ", - }, nil -} - -var dmc5 = &DMCache{ - PerDevice: true, - getCurrentStatus: badFormat, -} - -func TestDMCacheStats_5(t *testing.T) { +func TestBadFormatOfStatus(t *testing.T) { var acc testutil.Accumulator + var plugin = &DMCache{ + PerDevice: true, + getCurrentStatus: func() ([]string, error) { + return badFormatOutput, nil + }, + } - err := dmc5.Gather(&acc) + err := plugin.Gather(&acc) require.Error(t, err) } From 2091671c22ec08f2fc1f148511db9ba9c9acbb31 Mon Sep 17 00:00:00 2001 From: Vladimir S Date: Fri, 7 Apr 2017 13:13:06 +0300 Subject: [PATCH 09/18] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 333963bd59f85..46d8b57d5b1d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ be deprecated eventually. - [#2587](https://github.com/influxdata/telegraf/pull/2587): Add json timestamp units configurability - [#2597](https://github.com/influxdata/telegraf/issues/2597): Add support for Linux sysctl-fs metrics. - [#2425](https://github.com/influxdata/telegraf/pull/2425): Support to include/exclude docker container labels as tags +- [#1667](https://github.com/influxdata/telegraf/pull/1667): dmcache input plugin ### Bugfixes From edd631a585f1694ea799b84c6f3f1fd84a3e8651 Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Wed, 24 Aug 2016 22:41:53 +0300 Subject: [PATCH 10/18] Add dmcache input plugin --- plugins/inputs/all/all.go | 1 + plugins/inputs/dmcache/README.md | 11 ++ plugins/inputs/dmcache/dmcache.go | 49 ++++++++ plugins/inputs/dmcache/dmcache_linux.go | 131 +++++++++++++++++++++ plugins/inputs/dmcache/dmcache_notlinux.go | 11 ++ plugins/inputs/dmcache/dmcache_test.go | 94 +++++++++++++++ 6 files changed, 297 insertions(+) create mode 100644 plugins/inputs/dmcache/README.md create mode 100644 plugins/inputs/dmcache/dmcache.go create mode 100644 plugins/inputs/dmcache/dmcache_linux.go create mode 100644 plugins/inputs/dmcache/dmcache_notlinux.go create mode 100644 plugins/inputs/dmcache/dmcache_test.go diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index a9147c53ed153..983179e903bdc 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -15,6 +15,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/couchbase" _ "github.com/influxdata/telegraf/plugins/inputs/couchdb" _ "github.com/influxdata/telegraf/plugins/inputs/disque" + _ "github.com/influxdata/telegraf/plugins/inputs/dmcache" _ "github.com/influxdata/telegraf/plugins/inputs/dns_query" _ "github.com/influxdata/telegraf/plugins/inputs/docker" _ "github.com/influxdata/telegraf/plugins/inputs/dovecot" diff --git a/plugins/inputs/dmcache/README.md b/plugins/inputs/dmcache/README.md new file mode 100644 index 0000000000000..87931376e0e18 --- /dev/null +++ b/plugins/inputs/dmcache/README.md @@ -0,0 +1,11 @@ +# DMCache Input Plugin + +This plugin provide a native collection for dmsetup based statistics for dm-cache + +## Configuration + +``` +[[inputs.dmcache]] +## Whether to report per-device stats or not + per_device = true +``` diff --git a/plugins/inputs/dmcache/dmcache.go b/plugins/inputs/dmcache/dmcache.go new file mode 100644 index 0000000000000..bfcb6dadcb751 --- /dev/null +++ b/plugins/inputs/dmcache/dmcache.go @@ -0,0 +1,49 @@ +package dmcache + +import ( + "os/exec" + "strings" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +type DMCache struct { + PerDevice bool `toml:"per_device"` + rawStatus func() ([]string, error) +} + +var sampleConfig = ` + ## Whether to report per-device stats or not + per_device = true +` + +func (c *DMCache) SampleConfig() string { + return sampleConfig +} + +func (c *DMCache) Description() string { + return "Provide a native collection for dmsetup based statistics for dm-cache" +} + +func init() { + inputs.Add("dmcache", func() telegraf.Input { + return &DMCache{ + PerDevice: true, + rawStatus: func() ([]string, error) { + out, err := exec.Command("/bin/sh", "-c", "sudo /sbin/dmsetup status --target cache").Output() + if err != nil { + return nil, err + } + if string(out) == "No devices found\n" { + return nil, nil + } + + status := strings.Split(string(out), "\n") + status = status[:len(status)-1] // removing last empty line + + return status, nil + }, + } + }) +} diff --git a/plugins/inputs/dmcache/dmcache_linux.go b/plugins/inputs/dmcache/dmcache_linux.go new file mode 100644 index 0000000000000..ed8b88c6809d3 --- /dev/null +++ b/plugins/inputs/dmcache/dmcache_linux.go @@ -0,0 +1,131 @@ +// +build linux + +package dmcache + +import ( + "strconv" + "strings" + + "github.com/influxdata/telegraf" +) + +const metricName = "dmcache" + +var fieldNames = [...]string{ + "metadata_used", + "metadata_free", + "cache_used", + "cache_free", + "read_hits", + "read_misses", + "write_hits", + "write_misses", + "demotions", + "promotions", + "dirty", +} + +func (c *DMCache) Gather(acc telegraf.Accumulator) error { + outputLines, err := c.rawStatus() + if err != nil { + return err + } + + var total map[string]interface{} + if !c.PerDevice { + total = make(map[string]interface{}) + } + + for _, s := range outputLines { + fields := make(map[string]interface{}) + data, err := parseDMSetupStatus(s) + if err != nil { + return err + } + + for _, f := range fieldNames { + fields[f] = calculateSize(data, f) + } + + if c.PerDevice { + tags := map[string]string{"device": data["device"].(string)} + acc.AddFields(metricName, fields, tags) + } else { + aggregateStats(total, fields) + } + } + + if !c.PerDevice { + acc.AddFields(metricName, total, nil) + } + + return nil +} + +func parseDMSetupStatus(line string) (status map[string]interface{}, err error) { + defer func() { + if r := recover(); r != nil { + status = nil + err = r.(error) + } + }() + + values := strings.Split(line, " ") + status = make(map[string]interface{}) + + status["device"] = values[0][:len(values[0])-1] + status["length"] = toInt(values[2]) + status["target"] = values[3] + status["metadata_blocksize"] = toInt(values[4]) + status["metadata_used"] = toInt(strings.Split(values[5], "/")[0]) + status["metadata_total"] = toInt(strings.Split(values[5], "/")[1]) + status["cache_blocksize"] = toInt(values[6]) + status["cache_used"] = toInt(strings.Split(values[7], "/")[0]) + status["cache_total"] = toInt(strings.Split(values[7], "/")[1]) + status["read_hits"] = toInt(values[8]) + status["read_misses"] = toInt(values[9]) + status["write_hits"] = toInt(values[10]) + status["write_misses"] = toInt(values[11]) + status["demotions"] = toInt(values[12]) + status["promotions"] = toInt(values[13]) + status["dirty"] = toInt(values[14]) + status["blocksize"] = 512 + + return status, nil +} + +func toInt(s string) int { + i, err := strconv.Atoi(s) + if err != nil { + panic(err) + } + return i +} + +func calculateSize(data map[string]interface{}, key string) (value int) { + if key == "metadata_free" { + value = data["metadata_total"].(int) - data["metadata_used"].(int) + } else if key == "cache_free" { + value = data["cache_total"].(int) - data["cache_used"].(int) - data["dirty"].(int) + } else { + value = data[key].(int) + } + + if key == "metadata_free" || key == "metadata_used" { + value = value * data["blocksize"].(int) * data["metadata_blocksize"].(int) + } else { + value = value * data["blocksize"].(int) * data["cache_blocksize"].(int) + } + + return +} + +func aggregateStats(total, fields map[string]interface{}) { + for key, value := range fields { + if _, ok := total[key]; ok { + total[key] = total[key].(int) + value.(int) + } else { + total[key] = value.(int) + } + } +} diff --git a/plugins/inputs/dmcache/dmcache_notlinux.go b/plugins/inputs/dmcache/dmcache_notlinux.go new file mode 100644 index 0000000000000..478c2f163e4dc --- /dev/null +++ b/plugins/inputs/dmcache/dmcache_notlinux.go @@ -0,0 +1,11 @@ +// +build !linux + +package dmcache + +import ( + "github.com/influxdata/telegraf" +) + +func (c *DMCache) Gather(acc telegraf.Accumulator) error { + return nil +} diff --git a/plugins/inputs/dmcache/dmcache_test.go b/plugins/inputs/dmcache/dmcache_test.go new file mode 100644 index 0000000000000..42d5072990802 --- /dev/null +++ b/plugins/inputs/dmcache/dmcache_test.go @@ -0,0 +1,94 @@ +// +build linux + +package dmcache + +import ( + "testing" + + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func output2Devices() ([]string, error) { + return []string{ + "cs-1: 0 4883791872 cache 8 1018/1501122 512 7/464962 139 352643 15 46 0 7 0 1 writeback 2 migration_threshold 2048 mq 10 random_threshold 4 sequential_threshold 512 discard_promote_adjustment 1 read_promote_adjustment 4 write_promote_adjustment 8", + "cs-2: 0 4294967296 cache 8 72352/1310720 128 26/24327168 2409 286 265 524682 0 0 0 1 writethrough 2 migration_threshold 2048 mq 10 random_threshold 4 sequential_threshold 512 discard_promote_adjustment 1 read_promote_adjustment 4 write_promote_adjustment 8", + }, nil +} + +var dmc1 = &DMCache{ + PerDevice: true, + rawStatus: output2Devices, +} + +func TestDMCacheStats_1(t *testing.T) { + var acc testutil.Accumulator + + err := dmc1.Gather(&acc) + require.NoError(t, err) + + tags1 := map[string]string{ + "device": "cs-1", + } + fields1 := map[string]interface{}{ + "metadata_used": 4169728, + "metadata_free": 6144425984, + "cache_used": 1835008, + "cache_free": 121885163520, + "read_hits": 36438016, + "read_misses": 92443246592, + "write_hits": 3932160, + "write_misses": 12058624, + "demotions": 0, + "promotions": 1835008, + "dirty": 0, + } + acc.AssertContainsTaggedFields(t, "dmcache", fields1, tags1) + + tags2 := map[string]string{ + "device": "cs-2", + } + fields2 := map[string]interface{}{ + "metadata_used": 296353792, + "metadata_free": 5072355328, + "cache_used": 1703936, + "cache_free": 1594303578112, + "read_hits": 157876224, + "read_misses": 18743296, + "write_hits": 17367040, + "write_misses": 34385559552, + "demotions": 0, + "promotions": 0, + "dirty": 0, + } + acc.AssertContainsTaggedFields(t, "dmcache", fields2, tags2) +} + +var dmc2 = &DMCache{ + PerDevice: false, + rawStatus: output2Devices, +} + +func TestDMCacheStats_2(t *testing.T) { + var acc testutil.Accumulator + + err := dmc2.Gather(&acc) + require.NoError(t, err) + + tags := map[string]string{} + + fields := map[string]interface{}{ + "metadata_used": 300523520, + "metadata_free": 11216781312, + "cache_used": 3538944, + "cache_free": 1716188741632, + "read_hits": 194314240, + "read_misses": 92461989888, + "write_hits": 21299200, + "write_misses": 34397618176, + "demotions": 0, + "promotions": 1835008, + "dirty": 0, + } + acc.AssertContainsTaggedFields(t, "dmcache", fields, tags) +} From 525fc05915918ae4edfca12c1d8225e0032a1e72 Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Wed, 29 Mar 2017 12:19:25 +0300 Subject: [PATCH 11/18] dmcache refactoring --- plugins/inputs/dmcache/README.md | 10 ++- plugins/inputs/dmcache/dmcache.go | 4 +- plugins/inputs/dmcache/dmcache_linux.go | 109 +++++++++++++++--------- plugins/inputs/dmcache/dmcache_test.go | 25 +++++- 4 files changed, 99 insertions(+), 49 deletions(-) diff --git a/plugins/inputs/dmcache/README.md b/plugins/inputs/dmcache/README.md index 87931376e0e18..ae4142e9aa2fa 100644 --- a/plugins/inputs/dmcache/README.md +++ b/plugins/inputs/dmcache/README.md @@ -1,11 +1,15 @@ # DMCache Input Plugin -This plugin provide a native collection for dmsetup based statistics for dm-cache +This plugin provide a native collection for dmsetup based statistics for dm-cache. + +This plugin requires sudo, that is why you should setup and be sure that the telegraf is able to execute sudo without a password. + +`sudo /sbin/dmsetup status --target cache` is the full command that telegraf will run for debugging purposes. ## Configuration ``` [[inputs.dmcache]] -## Whether to report per-device stats or not - per_device = true + ## Whether to report per-device stats or not + per_device = true ``` diff --git a/plugins/inputs/dmcache/dmcache.go b/plugins/inputs/dmcache/dmcache.go index bfcb6dadcb751..c36eed636af77 100644 --- a/plugins/inputs/dmcache/dmcache.go +++ b/plugins/inputs/dmcache/dmcache.go @@ -14,8 +14,8 @@ type DMCache struct { } var sampleConfig = ` - ## Whether to report per-device stats or not - per_device = true + ## Whether to report per-device stats or not + per_device = true ` func (c *DMCache) SampleConfig() string { diff --git a/plugins/inputs/dmcache/dmcache_linux.go b/plugins/inputs/dmcache/dmcache_linux.go index ed8b88c6809d3..b64b19056a4fd 100644 --- a/plugins/inputs/dmcache/dmcache_linux.go +++ b/plugins/inputs/dmcache/dmcache_linux.go @@ -6,6 +6,8 @@ import ( "strconv" "strings" + "errors" + "github.com/influxdata/telegraf" ) @@ -31,10 +33,7 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { return err } - var total map[string]interface{} - if !c.PerDevice { - total = make(map[string]interface{}) - } + total := make(map[string]interface{}) for _, s := range outputLines { fields := make(map[string]interface{}) @@ -50,58 +49,86 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { if c.PerDevice { tags := map[string]string{"device": data["device"].(string)} acc.AddFields(metricName, fields, tags) - } else { - aggregateStats(total, fields) } + aggregateStats(total, fields) } - if !c.PerDevice { - acc.AddFields(metricName, total, nil) - } + acc.AddFields(metricName, total, map[string]string{"device": "all"}) return nil } -func parseDMSetupStatus(line string) (status map[string]interface{}, err error) { - defer func() { - if r := recover(); r != nil { - status = nil - err = r.(error) - } - }() - - values := strings.Split(line, " ") - status = make(map[string]interface{}) +func parseDMSetupStatus(line string) (map[string]interface{}, error) { + var err error + status := make(map[string]interface{}) + values := strings.Fields(line) + if len(values) < 15 { + return nil, errors.New("dmsetup status data have invalid format") + } status["device"] = values[0][:len(values[0])-1] - status["length"] = toInt(values[2]) + status["length"], err = strconv.Atoi(values[2]) + if err != nil { + return nil, err + } status["target"] = values[3] - status["metadata_blocksize"] = toInt(values[4]) - status["metadata_used"] = toInt(strings.Split(values[5], "/")[0]) - status["metadata_total"] = toInt(strings.Split(values[5], "/")[1]) - status["cache_blocksize"] = toInt(values[6]) - status["cache_used"] = toInt(strings.Split(values[7], "/")[0]) - status["cache_total"] = toInt(strings.Split(values[7], "/")[1]) - status["read_hits"] = toInt(values[8]) - status["read_misses"] = toInt(values[9]) - status["write_hits"] = toInt(values[10]) - status["write_misses"] = toInt(values[11]) - status["demotions"] = toInt(values[12]) - status["promotions"] = toInt(values[13]) - status["dirty"] = toInt(values[14]) + status["metadata_blocksize"], err = strconv.Atoi(values[4]) + if err != nil { + return nil, err + } + status["metadata_used"], err = strconv.Atoi(strings.Split(values[5], "/")[0]) + if err != nil { + return nil, err + } + status["metadata_total"], err = strconv.Atoi(strings.Split(values[5], "/")[1]) + if err != nil { + return nil, err + } + status["cache_blocksize"], err = strconv.Atoi(values[6]) + if err != nil { + return nil, err + } + status["cache_used"], err = strconv.Atoi(strings.Split(values[7], "/")[0]) + if err != nil { + return nil, err + } + status["cache_total"], err = strconv.Atoi(strings.Split(values[7], "/")[1]) + if err != nil { + return nil, err + } + status["read_hits"], err = strconv.Atoi(values[8]) + if err != nil { + return nil, err + } + status["read_misses"], err = strconv.Atoi(values[9]) + if err != nil { + return nil, err + } + status["write_hits"], err = strconv.Atoi(values[10]) + if err != nil { + return nil, err + } + status["write_misses"], err = strconv.Atoi(values[11]) + if err != nil { + return nil, err + } + status["demotions"], err = strconv.Atoi(values[12]) + if err != nil { + return nil, err + } + status["promotions"], err = strconv.Atoi(values[13]) + if err != nil { + return nil, err + } + status["dirty"], err = strconv.Atoi(values[14]) + if err != nil { + return nil, err + } status["blocksize"] = 512 return status, nil } -func toInt(s string) int { - i, err := strconv.Atoi(s) - if err != nil { - panic(err) - } - return i -} - func calculateSize(data map[string]interface{}, key string) (value int) { if key == "metadata_free" { value = data["metadata_total"].(int) - data["metadata_used"].(int) diff --git a/plugins/inputs/dmcache/dmcache_test.go b/plugins/inputs/dmcache/dmcache_test.go index 42d5072990802..d4295b38b268b 100644 --- a/plugins/inputs/dmcache/dmcache_test.go +++ b/plugins/inputs/dmcache/dmcache_test.go @@ -1,5 +1,3 @@ -// +build linux - package dmcache import ( @@ -62,6 +60,25 @@ func TestDMCacheStats_1(t *testing.T) { "dirty": 0, } acc.AssertContainsTaggedFields(t, "dmcache", fields2, tags2) + + tags3 := map[string]string{ + "device": "all", + } + + fields3 := map[string]interface{}{ + "metadata_used": 300523520, + "metadata_free": 11216781312, + "cache_used": 3538944, + "cache_free": 1716188741632, + "read_hits": 194314240, + "read_misses": 92461989888, + "write_hits": 21299200, + "write_misses": 34397618176, + "demotions": 0, + "promotions": 1835008, + "dirty": 0, + } + acc.AssertContainsTaggedFields(t, "dmcache", fields3, tags3) } var dmc2 = &DMCache{ @@ -75,7 +92,9 @@ func TestDMCacheStats_2(t *testing.T) { err := dmc2.Gather(&acc) require.NoError(t, err) - tags := map[string]string{} + tags := map[string]string{ + "device": "all", + } fields := map[string]interface{}{ "metadata_used": 300523520, From c7a7ab68e25b857cf14edd706bff77edb31cb152 Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Mon, 3 Apr 2017 16:02:59 +0300 Subject: [PATCH 12/18] refactoring and new tests --- plugins/inputs/dmcache/dmcache.go | 36 ++++++++------- plugins/inputs/dmcache/dmcache_linux.go | 23 +++++++--- plugins/inputs/dmcache/dmcache_test.go | 59 +++++++++++++++++++++++-- 3 files changed, 90 insertions(+), 28 deletions(-) diff --git a/plugins/inputs/dmcache/dmcache.go b/plugins/inputs/dmcache/dmcache.go index c36eed636af77..b933cc098b10a 100644 --- a/plugins/inputs/dmcache/dmcache.go +++ b/plugins/inputs/dmcache/dmcache.go @@ -9,8 +9,8 @@ import ( ) type DMCache struct { - PerDevice bool `toml:"per_device"` - rawStatus func() ([]string, error) + PerDevice bool `toml:"per_device"` + getCurrentStatus func() ([]string, error) } var sampleConfig = ` @@ -26,24 +26,26 @@ func (c *DMCache) Description() string { return "Provide a native collection for dmsetup based statistics for dm-cache" } +func dmSetupStatus() ([]string, error) { + out, err := exec.Command("/bin/sh", "-c", "sudo /sbin/dmsetup status --target cache").Output() + if err != nil { + return nil, err + } + if string(out) == "No devices found\n" { + return []string{}, nil + } + + outString := strings.TrimRight(string(out), "\n") + status := strings.Split(outString, "\n") + + return status, nil +} + func init() { inputs.Add("dmcache", func() telegraf.Input { return &DMCache{ - PerDevice: true, - rawStatus: func() ([]string, error) { - out, err := exec.Command("/bin/sh", "-c", "sudo /sbin/dmsetup status --target cache").Output() - if err != nil { - return nil, err - } - if string(out) == "No devices found\n" { - return nil, nil - } - - status := strings.Split(string(out), "\n") - status = status[:len(status)-1] // removing last empty line - - return status, nil - }, + PerDevice: true, + getCurrentStatus: dmSetupStatus, } }) } diff --git a/plugins/inputs/dmcache/dmcache_linux.go b/plugins/inputs/dmcache/dmcache_linux.go index b64b19056a4fd..12fc2a0919401 100644 --- a/plugins/inputs/dmcache/dmcache_linux.go +++ b/plugins/inputs/dmcache/dmcache_linux.go @@ -28,7 +28,7 @@ var fieldNames = [...]string{ } func (c *DMCache) Gather(acc telegraf.Accumulator) error { - outputLines, err := c.rawStatus() + outputLines, err := c.getCurrentStatus() if err != nil { return err } @@ -60,13 +60,14 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { func parseDMSetupStatus(line string) (map[string]interface{}, error) { var err error + parseError := errors.New("Output from dmsetup could not be parsed") status := make(map[string]interface{}) values := strings.Fields(line) if len(values) < 15 { - return nil, errors.New("dmsetup status data have invalid format") + return nil, parseError } - status["device"] = values[0][:len(values[0])-1] + status["device"] = strings.TrimRight(values[0], ":") status["length"], err = strconv.Atoi(values[2]) if err != nil { return nil, err @@ -76,11 +77,15 @@ func parseDMSetupStatus(line string) (map[string]interface{}, error) { if err != nil { return nil, err } - status["metadata_used"], err = strconv.Atoi(strings.Split(values[5], "/")[0]) + metadata := strings.Split(values[5], "/") + if len(metadata) != 2 { + return nil, parseError + } + status["metadata_used"], err = strconv.Atoi(metadata[0]) if err != nil { return nil, err } - status["metadata_total"], err = strconv.Atoi(strings.Split(values[5], "/")[1]) + status["metadata_total"], err = strconv.Atoi(metadata[1]) if err != nil { return nil, err } @@ -88,11 +93,15 @@ func parseDMSetupStatus(line string) (map[string]interface{}, error) { if err != nil { return nil, err } - status["cache_used"], err = strconv.Atoi(strings.Split(values[7], "/")[0]) + cache := strings.Split(values[7], "/") + if len(cache) != 2 { + return nil, parseError + } + status["cache_used"], err = strconv.Atoi(cache[0]) if err != nil { return nil, err } - status["cache_total"], err = strconv.Atoi(strings.Split(values[7], "/")[1]) + status["cache_total"], err = strconv.Atoi(cache[1]) if err != nil { return nil, err } diff --git a/plugins/inputs/dmcache/dmcache_test.go b/plugins/inputs/dmcache/dmcache_test.go index d4295b38b268b..b35dc0eede9f3 100644 --- a/plugins/inputs/dmcache/dmcache_test.go +++ b/plugins/inputs/dmcache/dmcache_test.go @@ -1,6 +1,7 @@ package dmcache import ( + "errors" "testing" "github.com/influxdata/telegraf/testutil" @@ -15,8 +16,8 @@ func output2Devices() ([]string, error) { } var dmc1 = &DMCache{ - PerDevice: true, - rawStatus: output2Devices, + PerDevice: true, + getCurrentStatus: output2Devices, } func TestDMCacheStats_1(t *testing.T) { @@ -82,8 +83,8 @@ func TestDMCacheStats_1(t *testing.T) { } var dmc2 = &DMCache{ - PerDevice: false, - rawStatus: output2Devices, + PerDevice: false, + getCurrentStatus: output2Devices, } func TestDMCacheStats_2(t *testing.T) { @@ -111,3 +112,53 @@ func TestDMCacheStats_2(t *testing.T) { } acc.AssertContainsTaggedFields(t, "dmcache", fields, tags) } + +func outputNoDevices() ([]string, error) { + return []string{}, nil +} + +var dmc3 = &DMCache{ + PerDevice: true, + getCurrentStatus: outputNoDevices, +} + +func TestDMCacheStats_3(t *testing.T) { + var acc testutil.Accumulator + + err := dmc3.Gather(&acc) + require.NoError(t, err) +} + +func noDMSetup() ([]string, error) { + return []string{}, errors.New("dmsetup doesn't exist") +} + +var dmc4 = &DMCache{ + PerDevice: true, + getCurrentStatus: noDMSetup, +} + +func TestDMCacheStats_4(t *testing.T) { + var acc testutil.Accumulator + + err := dmc4.Gather(&acc) + require.Error(t, err) +} + +func badFormat() ([]string, error) { + return []string{ + "cs-1: 0 4883791872 cache 8 1018/1501122 512 7/464962 139 352643 ", + }, nil +} + +var dmc5 = &DMCache{ + PerDevice: true, + getCurrentStatus: badFormat, +} + +func TestDMCacheStats_5(t *testing.T) { + var acc testutil.Accumulator + + err := dmc5.Gather(&acc) + require.Error(t, err) +} From b696320ff0b702aeec19c55863cb5ec8daae8ad3 Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Tue, 4 Apr 2017 17:40:52 +0300 Subject: [PATCH 13/18] removed synthesized values --- plugins/inputs/dmcache/dmcache_linux.go | 52 ++---------- plugins/inputs/dmcache/dmcache_test.go | 104 ++++++++++++++---------- 2 files changed, 68 insertions(+), 88 deletions(-) diff --git a/plugins/inputs/dmcache/dmcache_linux.go b/plugins/inputs/dmcache/dmcache_linux.go index 12fc2a0919401..84cece5684ab7 100644 --- a/plugins/inputs/dmcache/dmcache_linux.go +++ b/plugins/inputs/dmcache/dmcache_linux.go @@ -13,20 +13,6 @@ import ( const metricName = "dmcache" -var fieldNames = [...]string{ - "metadata_used", - "metadata_free", - "cache_used", - "cache_free", - "read_hits", - "read_misses", - "write_hits", - "write_misses", - "demotions", - "promotions", - "dirty", -} - func (c *DMCache) Gather(acc telegraf.Accumulator) error { outputLines, err := c.getCurrentStatus() if err != nil { @@ -36,18 +22,13 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { total := make(map[string]interface{}) for _, s := range outputLines { - fields := make(map[string]interface{}) - data, err := parseDMSetupStatus(s) + fields, err := parseDMSetupStatus(s) if err != nil { return err } - for _, f := range fieldNames { - fields[f] = calculateSize(data, f) - } - if c.PerDevice { - tags := map[string]string{"device": data["device"].(string)} + tags := map[string]string{"device": fields["device"].(string)} acc.AddFields(metricName, fields, tags) } aggregateStats(total, fields) @@ -133,35 +114,18 @@ func parseDMSetupStatus(line string) (map[string]interface{}, error) { if err != nil { return nil, err } - status["blocksize"] = 512 return status, nil } -func calculateSize(data map[string]interface{}, key string) (value int) { - if key == "metadata_free" { - value = data["metadata_total"].(int) - data["metadata_used"].(int) - } else if key == "cache_free" { - value = data["cache_total"].(int) - data["cache_used"].(int) - data["dirty"].(int) - } else { - value = data[key].(int) - } - - if key == "metadata_free" || key == "metadata_used" { - value = value * data["blocksize"].(int) * data["metadata_blocksize"].(int) - } else { - value = value * data["blocksize"].(int) * data["cache_blocksize"].(int) - } - - return -} - func aggregateStats(total, fields map[string]interface{}) { for key, value := range fields { - if _, ok := total[key]; ok { - total[key] = total[key].(int) + value.(int) - } else { - total[key] = value.(int) + if _, ok := value.(int); ok { + if _, ok := total[key]; ok { + total[key] = total[key].(int) + value.(int) + } else { + total[key] = value.(int) + } } } } diff --git a/plugins/inputs/dmcache/dmcache_test.go b/plugins/inputs/dmcache/dmcache_test.go index b35dc0eede9f3..2c317f9da16b7 100644 --- a/plugins/inputs/dmcache/dmcache_test.go +++ b/plugins/inputs/dmcache/dmcache_test.go @@ -30,17 +30,22 @@ func TestDMCacheStats_1(t *testing.T) { "device": "cs-1", } fields1 := map[string]interface{}{ - "metadata_used": 4169728, - "metadata_free": 6144425984, - "cache_used": 1835008, - "cache_free": 121885163520, - "read_hits": 36438016, - "read_misses": 92443246592, - "write_hits": 3932160, - "write_misses": 12058624, - "demotions": 0, - "promotions": 1835008, - "dirty": 0, + "device": "cs-1", + "length": 4883791872, + "target": "cache", + "metadata_blocksize": 8, + "metadata_used": 1018, + "metadata_total": 1501122, + "cache_blocksize": 512, + "cache_used": 7, + "cache_total": 464962, + "read_hits": 139, + "read_misses": 352643, + "write_hits": 15, + "write_misses": 46, + "demotions": 0, + "promotions": 7, + "dirty": 0, } acc.AssertContainsTaggedFields(t, "dmcache", fields1, tags1) @@ -48,17 +53,22 @@ func TestDMCacheStats_1(t *testing.T) { "device": "cs-2", } fields2 := map[string]interface{}{ - "metadata_used": 296353792, - "metadata_free": 5072355328, - "cache_used": 1703936, - "cache_free": 1594303578112, - "read_hits": 157876224, - "read_misses": 18743296, - "write_hits": 17367040, - "write_misses": 34385559552, - "demotions": 0, - "promotions": 0, - "dirty": 0, + "device": "cs-2", + "length": 4294967296, + "target": "cache", + "metadata_blocksize": 8, + "metadata_used": 72352, + "metadata_total": 1310720, + "cache_blocksize": 128, + "cache_used": 26, + "cache_total": 24327168, + "read_hits": 2409, + "read_misses": 286, + "write_hits": 265, + "write_misses": 524682, + "demotions": 0, + "promotions": 0, + "dirty": 0, } acc.AssertContainsTaggedFields(t, "dmcache", fields2, tags2) @@ -67,17 +77,20 @@ func TestDMCacheStats_1(t *testing.T) { } fields3 := map[string]interface{}{ - "metadata_used": 300523520, - "metadata_free": 11216781312, - "cache_used": 3538944, - "cache_free": 1716188741632, - "read_hits": 194314240, - "read_misses": 92461989888, - "write_hits": 21299200, - "write_misses": 34397618176, - "demotions": 0, - "promotions": 1835008, - "dirty": 0, + "length": 9178759168, + "metadata_blocksize": 16, + "metadata_used": 73370, + "metadata_total": 2811842, + "cache_blocksize": 640, + "cache_used": 33, + "cache_total": 24792130, + "read_hits": 2548, + "read_misses": 352929, + "write_hits": 280, + "write_misses": 524728, + "demotions": 0, + "promotions": 7, + "dirty": 0, } acc.AssertContainsTaggedFields(t, "dmcache", fields3, tags3) } @@ -98,17 +111,20 @@ func TestDMCacheStats_2(t *testing.T) { } fields := map[string]interface{}{ - "metadata_used": 300523520, - "metadata_free": 11216781312, - "cache_used": 3538944, - "cache_free": 1716188741632, - "read_hits": 194314240, - "read_misses": 92461989888, - "write_hits": 21299200, - "write_misses": 34397618176, - "demotions": 0, - "promotions": 1835008, - "dirty": 0, + "length": 9178759168, + "metadata_blocksize": 16, + "metadata_used": 73370, + "metadata_total": 2811842, + "cache_blocksize": 640, + "cache_used": 33, + "cache_total": 24792130, + "read_hits": 2548, + "read_misses": 352929, + "write_hits": 280, + "write_misses": 524728, + "demotions": 0, + "promotions": 7, + "dirty": 0, } acc.AssertContainsTaggedFields(t, "dmcache", fields, tags) } From 9044638fdb2daf5be1f2094836b7d0b0c2d66ef3 Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Wed, 5 Apr 2017 11:25:19 +0300 Subject: [PATCH 14/18] updated readme --- plugins/inputs/dmcache/README.md | 36 ++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/dmcache/README.md b/plugins/inputs/dmcache/README.md index ae4142e9aa2fa..8e45726860dc2 100644 --- a/plugins/inputs/dmcache/README.md +++ b/plugins/inputs/dmcache/README.md @@ -6,10 +6,42 @@ This plugin requires sudo, that is why you should setup and be sure that the tel `sudo /sbin/dmsetup status --target cache` is the full command that telegraf will run for debugging purposes. -## Configuration +### Configuration -``` +```toml [[inputs.dmcache]] ## Whether to report per-device stats or not per_device = true ``` + +### Measurements & Fields: + +- dmcache + - length + - target + - metadata_blocksize + - metadata_used + - metadata_total + - cache_blocksize + - cache_used + - cache_total + - read_hits + - read_misses + - write_hits + - write_misses + - demotions + - promotions + - dirty + +### Tags: + +- All measurements have the following tags: + - device + +### Example Output: + +``` +$ ./telegraf --test --config /etc/telegraf/telegraf.conf --input-filter dmcache +* Plugin: inputs.dmcache, Collection 1 +> dmcache,bu=linux,cls=server,dc=colo,device=vg02-splunk_data_lv,env=production,host=hostname,sr=splunk_indexer,trd=false cache_free=0i,cache_used=1099511627776i,demotions=3265223720960i,dirty=0i,metadata_free=1060880384i,metadata_used=12861440i,promotions=3265223720960i,read_hits=995134034411520i,read_misses=916807089127424i,write_hits=195107267543040i,write_misses=563725346013184i 1491362011000000000 +``` \ No newline at end of file From 6ab3f047a7905a224cb341314e9fc8f08f8b1442 Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Wed, 5 Apr 2017 12:08:41 +0300 Subject: [PATCH 15/18] refactored map to struct --- plugins/inputs/dmcache/dmcache.go | 18 ---- plugins/inputs/dmcache/dmcache_linux.go | 135 ++++++++++++++++-------- plugins/inputs/dmcache/dmcache_test.go | 4 - 3 files changed, 92 insertions(+), 65 deletions(-) diff --git a/plugins/inputs/dmcache/dmcache.go b/plugins/inputs/dmcache/dmcache.go index b933cc098b10a..25a398194edf8 100644 --- a/plugins/inputs/dmcache/dmcache.go +++ b/plugins/inputs/dmcache/dmcache.go @@ -1,9 +1,6 @@ package dmcache import ( - "os/exec" - "strings" - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" ) @@ -26,21 +23,6 @@ func (c *DMCache) Description() string { return "Provide a native collection for dmsetup based statistics for dm-cache" } -func dmSetupStatus() ([]string, error) { - out, err := exec.Command("/bin/sh", "-c", "sudo /sbin/dmsetup status --target cache").Output() - if err != nil { - return nil, err - } - if string(out) == "No devices found\n" { - return []string{}, nil - } - - outString := strings.TrimRight(string(out), "\n") - status := strings.Split(outString, "\n") - - return status, nil -} - func init() { inputs.Add("dmcache", func() telegraf.Input { return &DMCache{ diff --git a/plugins/inputs/dmcache/dmcache_linux.go b/plugins/inputs/dmcache/dmcache_linux.go index 84cece5684ab7..8c698834b384f 100644 --- a/plugins/inputs/dmcache/dmcache_linux.go +++ b/plugins/inputs/dmcache/dmcache_linux.go @@ -3,6 +3,7 @@ package dmcache import ( + "os/exec" "strconv" "strings" @@ -13,6 +14,25 @@ import ( const metricName = "dmcache" +type cacheStatus struct { + device string + length int + target string + metadataBlocksize int + metadataUsed int + metadataTotal int + cacheBlocksize int + cacheUsed int + cacheTotal int + readHits int + readMisses int + writeHits int + writeMisses int + demotions int + promotions int + dirty int +} + func (c *DMCache) Gather(acc telegraf.Accumulator) error { outputLines, err := c.getCurrentStatus() if err != nil { @@ -22,13 +42,29 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { total := make(map[string]interface{}) for _, s := range outputLines { - fields, err := parseDMSetupStatus(s) + status, err := parseDMSetupStatus(s) if err != nil { return err } + fields := make(map[string]interface{}) + fields["length"] = status.length + fields["metadata_blocksize"] = status.metadataBlocksize + fields["metadata_used"] = status.metadataUsed + fields["metadata_total"] = status.metadataTotal + fields["cache_blocksize"] = status.cacheBlocksize + fields["cache_used"] = status.cacheUsed + fields["cache_total"] = status.cacheTotal + fields["read_hits"] = status.readHits + fields["read_misses"] = status.readMisses + fields["write_hits"] = status.writeHits + fields["write_misses"] = status.writeMisses + fields["demotions"] = status.demotions + fields["promotions"] = status.promotions + fields["dirty"] = status.dirty + if c.PerDevice { - tags := map[string]string{"device": fields["device"].(string)} + tags := map[string]string{"device": status.device} acc.AddFields(metricName, fields, tags) } aggregateStats(total, fields) @@ -39,80 +75,80 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { return nil } -func parseDMSetupStatus(line string) (map[string]interface{}, error) { +func parseDMSetupStatus(line string) (cacheStatus, error) { var err error parseError := errors.New("Output from dmsetup could not be parsed") - status := make(map[string]interface{}) + status := cacheStatus{} values := strings.Fields(line) if len(values) < 15 { - return nil, parseError + return cacheStatus{}, parseError } - status["device"] = strings.TrimRight(values[0], ":") - status["length"], err = strconv.Atoi(values[2]) + status.device = strings.TrimRight(values[0], ":") + status.length, err = strconv.Atoi(values[2]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["target"] = values[3] - status["metadata_blocksize"], err = strconv.Atoi(values[4]) + status.target = values[3] + status.metadataBlocksize, err = strconv.Atoi(values[4]) if err != nil { - return nil, err + return cacheStatus{}, err } metadata := strings.Split(values[5], "/") if len(metadata) != 2 { - return nil, parseError + return cacheStatus{}, parseError } - status["metadata_used"], err = strconv.Atoi(metadata[0]) + status.metadataUsed, err = strconv.Atoi(metadata[0]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["metadata_total"], err = strconv.Atoi(metadata[1]) + status.metadataTotal, err = strconv.Atoi(metadata[1]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["cache_blocksize"], err = strconv.Atoi(values[6]) + status.cacheBlocksize, err = strconv.Atoi(values[6]) if err != nil { - return nil, err + return cacheStatus{}, err } cache := strings.Split(values[7], "/") if len(cache) != 2 { - return nil, parseError + return cacheStatus{}, parseError } - status["cache_used"], err = strconv.Atoi(cache[0]) + status.cacheUsed, err = strconv.Atoi(cache[0]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["cache_total"], err = strconv.Atoi(cache[1]) + status.cacheTotal, err = strconv.Atoi(cache[1]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["read_hits"], err = strconv.Atoi(values[8]) + status.readHits, err = strconv.Atoi(values[8]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["read_misses"], err = strconv.Atoi(values[9]) + status.readMisses, err = strconv.Atoi(values[9]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["write_hits"], err = strconv.Atoi(values[10]) + status.writeHits, err = strconv.Atoi(values[10]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["write_misses"], err = strconv.Atoi(values[11]) + status.writeMisses, err = strconv.Atoi(values[11]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["demotions"], err = strconv.Atoi(values[12]) + status.demotions, err = strconv.Atoi(values[12]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["promotions"], err = strconv.Atoi(values[13]) + status.promotions, err = strconv.Atoi(values[13]) if err != nil { - return nil, err + return cacheStatus{}, err } - status["dirty"], err = strconv.Atoi(values[14]) + status.dirty, err = strconv.Atoi(values[14]) if err != nil { - return nil, err + return cacheStatus{}, err } return status, nil @@ -120,12 +156,25 @@ func parseDMSetupStatus(line string) (map[string]interface{}, error) { func aggregateStats(total, fields map[string]interface{}) { for key, value := range fields { - if _, ok := value.(int); ok { - if _, ok := total[key]; ok { - total[key] = total[key].(int) + value.(int) - } else { - total[key] = value.(int) - } + if _, ok := total[key]; ok { + total[key] = total[key].(int) + value.(int) + } else { + total[key] = value.(int) } } } + +func dmSetupStatus() ([]string, error) { + out, err := exec.Command("/bin/sh", "-c", "sudo /sbin/dmsetup status --target cache").Output() + if err != nil { + return nil, err + } + if string(out) == "No devices found\n" { + return []string{}, nil + } + + outString := strings.TrimRight(string(out), "\n") + status := strings.Split(outString, "\n") + + return status, nil +} diff --git a/plugins/inputs/dmcache/dmcache_test.go b/plugins/inputs/dmcache/dmcache_test.go index 2c317f9da16b7..e30b91e2ee6f4 100644 --- a/plugins/inputs/dmcache/dmcache_test.go +++ b/plugins/inputs/dmcache/dmcache_test.go @@ -30,9 +30,7 @@ func TestDMCacheStats_1(t *testing.T) { "device": "cs-1", } fields1 := map[string]interface{}{ - "device": "cs-1", "length": 4883791872, - "target": "cache", "metadata_blocksize": 8, "metadata_used": 1018, "metadata_total": 1501122, @@ -53,9 +51,7 @@ func TestDMCacheStats_1(t *testing.T) { "device": "cs-2", } fields2 := map[string]interface{}{ - "device": "cs-2", "length": 4294967296, - "target": "cache", "metadata_blocksize": 8, "metadata_used": 72352, "metadata_total": 1310720, From 95e01e058e8ed872e99ee5567dcbf207690d0a0e Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Thu, 6 Apr 2017 15:54:26 +0300 Subject: [PATCH 16/18] refactoring: switching to fields at the last moment --- plugins/inputs/dmcache/README.md | 4 +- plugins/inputs/dmcache/dmcache_linux.go | 66 ++++++++++++++----------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/plugins/inputs/dmcache/README.md b/plugins/inputs/dmcache/README.md index 8e45726860dc2..536d3f518bcaa 100644 --- a/plugins/inputs/dmcache/README.md +++ b/plugins/inputs/dmcache/README.md @@ -43,5 +43,5 @@ This plugin requires sudo, that is why you should setup and be sure that the tel ``` $ ./telegraf --test --config /etc/telegraf/telegraf.conf --input-filter dmcache * Plugin: inputs.dmcache, Collection 1 -> dmcache,bu=linux,cls=server,dc=colo,device=vg02-splunk_data_lv,env=production,host=hostname,sr=splunk_indexer,trd=false cache_free=0i,cache_used=1099511627776i,demotions=3265223720960i,dirty=0i,metadata_free=1060880384i,metadata_used=12861440i,promotions=3265223720960i,read_hits=995134034411520i,read_misses=916807089127424i,write_hits=195107267543040i,write_misses=563725346013184i 1491362011000000000 -``` \ No newline at end of file +> dmcache,device=example cache_blocksize=0i,read_hits=995134034411520i,read_misses=916807089127424i,write_hits=195107267543040i,metadata_used=12861440i,write_misses=563725346013184i,promotions=3265223720960i,dirty=0i,metadata_blocksize=0i,cache_used=1099511627776ii,cache_total=0i,length=0i,metadata_total=1073741824i,demotions=3265223720960i 1491482035000000000 +``` diff --git a/plugins/inputs/dmcache/dmcache_linux.go b/plugins/inputs/dmcache/dmcache_linux.go index 8c698834b384f..7ac1c96cae0f1 100644 --- a/plugins/inputs/dmcache/dmcache_linux.go +++ b/plugins/inputs/dmcache/dmcache_linux.go @@ -39,7 +39,7 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { return err } - total := make(map[string]interface{}) + totalStatus := cacheStatus{} for _, s := range outputLines { status, err := parseDMSetupStatus(s) @@ -47,30 +47,14 @@ func (c *DMCache) Gather(acc telegraf.Accumulator) error { return err } - fields := make(map[string]interface{}) - fields["length"] = status.length - fields["metadata_blocksize"] = status.metadataBlocksize - fields["metadata_used"] = status.metadataUsed - fields["metadata_total"] = status.metadataTotal - fields["cache_blocksize"] = status.cacheBlocksize - fields["cache_used"] = status.cacheUsed - fields["cache_total"] = status.cacheTotal - fields["read_hits"] = status.readHits - fields["read_misses"] = status.readMisses - fields["write_hits"] = status.writeHits - fields["write_misses"] = status.writeMisses - fields["demotions"] = status.demotions - fields["promotions"] = status.promotions - fields["dirty"] = status.dirty - if c.PerDevice { tags := map[string]string{"device": status.device} - acc.AddFields(metricName, fields, tags) + acc.AddFields(metricName, toFields(status), tags) } - aggregateStats(total, fields) + aggregateStats(&totalStatus, status) } - acc.AddFields(metricName, total, map[string]string{"device": "all"}) + acc.AddFields(metricName, toFields(totalStatus), map[string]string{"device": "all"}) return nil } @@ -154,14 +138,40 @@ func parseDMSetupStatus(line string) (cacheStatus, error) { return status, nil } -func aggregateStats(total, fields map[string]interface{}) { - for key, value := range fields { - if _, ok := total[key]; ok { - total[key] = total[key].(int) + value.(int) - } else { - total[key] = value.(int) - } - } +func aggregateStats(totalStatus *cacheStatus, status cacheStatus) { + totalStatus.length += status.length + totalStatus.metadataBlocksize += status.metadataBlocksize + totalStatus.metadataUsed += status.metadataUsed + totalStatus.metadataTotal += status.metadataTotal + totalStatus.cacheBlocksize += status.cacheBlocksize + totalStatus.cacheUsed += status.cacheUsed + totalStatus.cacheTotal += status.cacheTotal + totalStatus.readHits += status.readHits + totalStatus.readMisses += status.readMisses + totalStatus.writeHits += status.writeHits + totalStatus.writeMisses += status.writeMisses + totalStatus.demotions += status.demotions + totalStatus.promotions += status.promotions + totalStatus.dirty += status.dirty +} + +func toFields(status cacheStatus) map[string]interface{} { + fields := make(map[string]interface{}) + fields["length"] = status.length + fields["metadata_blocksize"] = status.metadataBlocksize + fields["metadata_used"] = status.metadataUsed + fields["metadata_total"] = status.metadataTotal + fields["cache_blocksize"] = status.cacheBlocksize + fields["cache_used"] = status.cacheUsed + fields["cache_total"] = status.cacheTotal + fields["read_hits"] = status.readHits + fields["read_misses"] = status.readMisses + fields["write_hits"] = status.writeHits + fields["write_misses"] = status.writeMisses + fields["demotions"] = status.demotions + fields["promotions"] = status.promotions + fields["dirty"] = status.dirty + return fields } func dmSetupStatus() ([]string, error) { From 798175764c822f1003133f377dc7ca96f1d9fc98 Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Fri, 7 Apr 2017 11:49:27 +0300 Subject: [PATCH 17/18] tests refactoring --- plugins/inputs/dmcache/dmcache_test.go | 107 ++++++++++++------------- 1 file changed, 50 insertions(+), 57 deletions(-) diff --git a/plugins/inputs/dmcache/dmcache_test.go b/plugins/inputs/dmcache/dmcache_test.go index e30b91e2ee6f4..c5989c413d9c0 100644 --- a/plugins/inputs/dmcache/dmcache_test.go +++ b/plugins/inputs/dmcache/dmcache_test.go @@ -8,22 +8,25 @@ import ( "github.com/stretchr/testify/require" ) -func output2Devices() ([]string, error) { - return []string{ +var ( + measurement = "dmcache" + badFormatOutput = []string{"cs-1: 0 4883791872 cache 8 1018/1501122 512 7/464962 139 352643 "} + good2DevicesFormatOutput = []string{ "cs-1: 0 4883791872 cache 8 1018/1501122 512 7/464962 139 352643 15 46 0 7 0 1 writeback 2 migration_threshold 2048 mq 10 random_threshold 4 sequential_threshold 512 discard_promote_adjustment 1 read_promote_adjustment 4 write_promote_adjustment 8", "cs-2: 0 4294967296 cache 8 72352/1310720 128 26/24327168 2409 286 265 524682 0 0 0 1 writethrough 2 migration_threshold 2048 mq 10 random_threshold 4 sequential_threshold 512 discard_promote_adjustment 1 read_promote_adjustment 4 write_promote_adjustment 8", - }, nil -} - -var dmc1 = &DMCache{ - PerDevice: true, - getCurrentStatus: output2Devices, -} + } +) -func TestDMCacheStats_1(t *testing.T) { +func TestPerDeviceGoodOutput(t *testing.T) { var acc testutil.Accumulator + var plugin = &DMCache{ + PerDevice: true, + getCurrentStatus: func() ([]string, error) { + return good2DevicesFormatOutput, nil + }, + } - err := dmc1.Gather(&acc) + err := plugin.Gather(&acc) require.NoError(t, err) tags1 := map[string]string{ @@ -45,7 +48,7 @@ func TestDMCacheStats_1(t *testing.T) { "promotions": 7, "dirty": 0, } - acc.AssertContainsTaggedFields(t, "dmcache", fields1, tags1) + acc.AssertContainsTaggedFields(t, measurement, fields1, tags1) tags2 := map[string]string{ "device": "cs-2", @@ -66,7 +69,7 @@ func TestDMCacheStats_1(t *testing.T) { "promotions": 0, "dirty": 0, } - acc.AssertContainsTaggedFields(t, "dmcache", fields2, tags2) + acc.AssertContainsTaggedFields(t, measurement, fields2, tags2) tags3 := map[string]string{ "device": "all", @@ -88,18 +91,19 @@ func TestDMCacheStats_1(t *testing.T) { "promotions": 7, "dirty": 0, } - acc.AssertContainsTaggedFields(t, "dmcache", fields3, tags3) + acc.AssertContainsTaggedFields(t, measurement, fields3, tags3) } -var dmc2 = &DMCache{ - PerDevice: false, - getCurrentStatus: output2Devices, -} - -func TestDMCacheStats_2(t *testing.T) { +func TestNotPerDeviceGoodOutput(t *testing.T) { var acc testutil.Accumulator + var plugin = &DMCache{ + PerDevice: false, + getCurrentStatus: func() ([]string, error) { + return good2DevicesFormatOutput, nil + }, + } - err := dmc2.Gather(&acc) + err := plugin.Gather(&acc) require.NoError(t, err) tags := map[string]string{ @@ -122,55 +126,44 @@ func TestDMCacheStats_2(t *testing.T) { "promotions": 7, "dirty": 0, } - acc.AssertContainsTaggedFields(t, "dmcache", fields, tags) -} - -func outputNoDevices() ([]string, error) { - return []string{}, nil -} - -var dmc3 = &DMCache{ - PerDevice: true, - getCurrentStatus: outputNoDevices, + acc.AssertContainsTaggedFields(t, measurement, fields, tags) } -func TestDMCacheStats_3(t *testing.T) { +func TestNoDevicesOutput(t *testing.T) { var acc testutil.Accumulator + var plugin = &DMCache{ + PerDevice: true, + getCurrentStatus: func() ([]string, error) { + return []string{}, nil + }, + } - err := dmc3.Gather(&acc) + err := plugin.Gather(&acc) require.NoError(t, err) } -func noDMSetup() ([]string, error) { - return []string{}, errors.New("dmsetup doesn't exist") -} - -var dmc4 = &DMCache{ - PerDevice: true, - getCurrentStatus: noDMSetup, -} - -func TestDMCacheStats_4(t *testing.T) { +func TestErrorDuringGettingStatus(t *testing.T) { var acc testutil.Accumulator + var plugin = &DMCache{ + PerDevice: true, + getCurrentStatus: func() ([]string, error) { + return nil, errors.New("dmsetup doesn't exist") + }, + } - err := dmc4.Gather(&acc) + err := plugin.Gather(&acc) require.Error(t, err) } -func badFormat() ([]string, error) { - return []string{ - "cs-1: 0 4883791872 cache 8 1018/1501122 512 7/464962 139 352643 ", - }, nil -} - -var dmc5 = &DMCache{ - PerDevice: true, - getCurrentStatus: badFormat, -} - -func TestDMCacheStats_5(t *testing.T) { +func TestBadFormatOfStatus(t *testing.T) { var acc testutil.Accumulator + var plugin = &DMCache{ + PerDevice: true, + getCurrentStatus: func() ([]string, error) { + return badFormatOutput, nil + }, + } - err := dmc5.Gather(&acc) + err := plugin.Gather(&acc) require.Error(t, err) } From bc220fbb9b6ec502dd03fd49561d6cb6261096bd Mon Sep 17 00:00:00 2001 From: Vladimir Sagan Date: Sat, 8 Apr 2017 00:18:04 +0300 Subject: [PATCH 18/18] added stub dmSetupStatus func for not linux build --- plugins/inputs/dmcache/dmcache_notlinux.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/inputs/dmcache/dmcache_notlinux.go b/plugins/inputs/dmcache/dmcache_notlinux.go index 478c2f163e4dc..ee1065638cab7 100644 --- a/plugins/inputs/dmcache/dmcache_notlinux.go +++ b/plugins/inputs/dmcache/dmcache_notlinux.go @@ -9,3 +9,7 @@ import ( func (c *DMCache) Gather(acc telegraf.Accumulator) error { return nil } + +func dmSetupStatus() ([]string, error) { + return []string{}, nil +}