From 1e1e2e15737629dc06513fa4e00fa12472cba266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=BA=A2?= Date: Wed, 18 Jul 2018 10:28:51 +0800 Subject: [PATCH] pandora_sender inner set pandora key convert --- sender/pandora/pandora.go | 3 ++ sender/pandora/pandora_test.go | 11 +++++-- transforms/mutate/pandorakey_convert.go | 14 +------- utils/models/models.go | 17 ---------- utils/models/utils.go | 29 ++++++++++++++++ utils/models/utils_test.go | 44 +++++++++++++++++++++++++ 6 files changed, 85 insertions(+), 33 deletions(-) diff --git a/sender/pandora/pandora.go b/sender/pandora/pandora.go index 1c2609450..bfd89c5a3 100644 --- a/sender/pandora/pandora.go +++ b/sender/pandora/pandora.go @@ -846,6 +846,9 @@ func (s *Sender) checkSchemaUpdate() { } func (s *Sender) Send(datas []Data) (se error) { + for i, v := range datas { + datas[i] = DeepConvertKey(v) + } switch s.sendType { case SendTypeRaw: return s.rawSend(datas) diff --git a/sender/pandora/pandora_test.go b/sender/pandora/pandora_test.go index 06ae76a34..f6bd5b836 100644 --- a/sender/pandora/pandora_test.go +++ b/sender/pandora/pandora_test.go @@ -1025,8 +1025,11 @@ func TestPandoraExtraInfo(t *testing.T) { if err != nil { t.Fatal(err) } - d = Data{} - d["x1"] = "123.2" + d = Data{ + "*x1": "123.2", + "x2.dot": "123.2", + "@timestamp": "2018-07-18T10:17:36.549054846+08:00", + } err = s.Send([]Data{d}) if st, ok := err.(*StatsError); ok { err = st.ErrorDetail @@ -1035,5 +1038,7 @@ func TestPandoraExtraInfo(t *testing.T) { t.Error(err) } resp = pandora.Body - assert.Equal(t, resp, "x1=123.2") + assert.Equal(t, true, strings.Contains(resp, "x1=123.2")) + assert.Equal(t, true, strings.Contains(resp, "x2_dot=123.2")) + assert.Equal(t, true, strings.Contains(resp, "timestamp=2018-07-18T10:17:36.549054846+08:00")) } diff --git a/transforms/mutate/pandorakey_convert.go b/transforms/mutate/pandorakey_convert.go index f832f78f7..4fcdd5500 100644 --- a/transforms/mutate/pandorakey_convert.go +++ b/transforms/mutate/pandorakey_convert.go @@ -22,25 +22,13 @@ func (g *PandoraKeyConvert) RawTransform(datas []string) ([]string, error) { func (g *PandoraKeyConvert) Transform(datas []Data) ([]Data, error) { for i, v := range datas { - datas[i] = deepConvertKey(v) + datas[i] = DeepConvertKey(v) } g.stats, _ = transforms.SetStatsInfo(nil, g.stats, 0, int64(len(datas)), g.Type()) return datas, nil } -func deepConvertKey(data map[string]interface{}) map[string]interface{} { - newData := make(map[string]interface{}) - for k, v := range data { - nk := PandoraKey(k) - if nv, ok := v.(map[string]interface{}); ok { - v = deepConvertKey(nv) - } - newData[nk] = v - } - return newData -} - func (g *PandoraKeyConvert) Description() string { //return "pandora_key_convert can convert data key name to valid pandora key" return "将数据中的key名称中不合Pandora字段名规则的字符转为下划线, 如 a.b/c 改为 a_b_c" diff --git a/utils/models/models.go b/utils/models/models.go index f36d40115..4baf295d4 100644 --- a/utils/models/models.go +++ b/utils/models/models.go @@ -146,20 +146,3 @@ func (se *StatsError) ErrorIndexIn(idx int) bool { } return false } - -func PandoraKey(key string) string { - var nk string - for _, c := range key { - if c >= '0' && c <= '9' { - if len(nk) == 0 { - nk = "K" - } - nk = nk + string(c) - } else if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') { - nk = nk + string(c) - } else if len(nk) > 0 { - nk = nk + "_" - } - } - return nk -} diff --git a/utils/models/utils.go b/utils/models/utils.go index c28ac7534..4729443e0 100644 --- a/utils/models/utils.go +++ b/utils/models/utils.go @@ -751,3 +751,32 @@ func GetMapList(data string) map[string]string { } return ret } + +func PandoraKey(key string) string { + var nk string + for _, c := range key { + if c >= '0' && c <= '9' { + if len(nk) == 0 { + nk = "K" + } + nk = nk + string(c) + } else if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') { + nk = nk + string(c) + } else if len(nk) > 0 { + nk = nk + "_" + } + } + return nk +} + +func DeepConvertKey(data map[string]interface{}) map[string]interface{} { + newData := make(map[string]interface{}) + for k, v := range data { + nk := PandoraKey(k) + if nv, ok := v.(map[string]interface{}); ok { + v = DeepConvertKey(nv) + } + newData[nk] = v + } + return newData +} diff --git a/utils/models/utils_test.go b/utils/models/utils_test.go index e6a2a092e..bb7f70039 100644 --- a/utils/models/utils_test.go +++ b/utils/models/utils_test.go @@ -614,3 +614,47 @@ func TestPickMapValue(t *testing.T) { PickMapValue(m, pick, "multi", "otherword") assert.NotEqual(t, exp, pick) } + +func TestPandoraKey(t *testing.T) { + testKeys := []string{"@timestamp", ".dot", "percent%100", "^^^^^^^^^^", "timestamp"} + expectKeys := []string{"timestamp", "dot", "percent_100", "", "timestamp"} + for idx, key := range testKeys { + actual := PandoraKey(key) + assert.Equal(t, expectKeys[idx], actual) + } +} + +func TestDeepConvertKey(t *testing.T) { + testDatas := []map[string]interface{}{ + { + "@timestamp": "2018-07-18T10:17:36.549054846+08:00", + //"timestamp": "2018-07-19T10:17:36.549054846+08:00", + }, + { + ".dot": "dot", + }, + { + "dot": "dot", + "percent%100": 100, + "^^^^^^^^^^": "mytest", + }, + } + expectDatas := []map[string]interface{}{ + { + "timestamp": "2018-07-18T10:17:36.549054846+08:00", + }, + { + "dot": "dot", + }, + { + "dot": "dot", + "percent_100": 100, + "": "mytest", + }, + } + + for idx, data := range testDatas { + actual := DeepConvertKey(data) + assert.Equal(t, expectDatas[idx], actual) + } +}