Merge pull request #611 from redHJ/pdr-6891

pandora_sender inner set pandora key convert
qiniu · Jul 18, 2018 · 43a41ce · 43a41ce
2 parents d73c47b + 1e1e2e1
commit 43a41ce
Show file tree

Hide file tree

Showing 6 changed files with 85 additions and 33 deletions.
diff --git a/sender/pandora/pandora.go b/sender/pandora/pandora.go
@@ -846,6 +846,9 @@ func (s *Sender) checkSchemaUpdate() {
 }
 
 func (s *Sender) Send(datas []Data) (se error) {
+	for i, v := range datas {
+		datas[i] = DeepConvertKey(v)
+	}
 	switch s.sendType {
 	case SendTypeRaw:
 		return s.rawSend(datas)

diff --git a/sender/pandora/pandora_test.go b/sender/pandora/pandora_test.go
@@ -1025,8 +1025,11 @@ func TestPandoraExtraInfo(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d = Data{}
-	d["x1"] = "123.2"
+	d = Data{
+		"*x1":        "123.2",
+		"x2.dot":     "123.2",
+		"@timestamp": "2018-07-18T10:17:36.549054846+08:00",
+	}
 	err = s.Send([]Data{d})
 	if st, ok := err.(*StatsError); ok {
 		err = st.ErrorDetail
@@ -1035,5 +1038,7 @@ func TestPandoraExtraInfo(t *testing.T) {
 		t.Error(err)
 	}
 	resp = pandora.Body
-	assert.Equal(t, resp, "x1=123.2")
+	assert.Equal(t, true, strings.Contains(resp, "x1=123.2"))
+	assert.Equal(t, true, strings.Contains(resp, "x2_dot=123.2"))
+	assert.Equal(t, true, strings.Contains(resp, "timestamp=2018-07-18T10:17:36.549054846+08:00"))
 }
diff --git a/transforms/mutate/pandorakey_convert.go b/transforms/mutate/pandorakey_convert.go
@@ -22,25 +22,13 @@ func (g *PandoraKeyConvert) RawTransform(datas []string) ([]string, error) {
 
 func (g *PandoraKeyConvert) Transform(datas []Data) ([]Data, error) {
 	for i, v := range datas {
-		datas[i] = deepConvertKey(v)
+		datas[i] = DeepConvertKey(v)
 	}
 
 	g.stats, _ = transforms.SetStatsInfo(nil, g.stats, 0, int64(len(datas)), g.Type())
 	return datas, nil
 }
 
-func deepConvertKey(data map[string]interface{}) map[string]interface{} {
-	newData := make(map[string]interface{})
-	for k, v := range data {
-		nk := PandoraKey(k)
-		if nv, ok := v.(map[string]interface{}); ok {
-			v = deepConvertKey(nv)
-		}
-		newData[nk] = v
-	}
-	return newData
-}
-
 func (g *PandoraKeyConvert) Description() string {
 	//return "pandora_key_convert can convert data key name to valid pandora key"
 	return "将数据中的key名称中不合Pandora字段名规则的字符转为下划线, 如 a.b/c 改为 a_b_c"

diff --git a/utils/models/models.go b/utils/models/models.go
@@ -146,20 +146,3 @@ func (se *StatsError) ErrorIndexIn(idx int) bool {
 	}
 	return false
 }
-
-func PandoraKey(key string) string {
-	var nk string
-	for _, c := range key {
-		if c >= '0' && c <= '9' {
-			if len(nk) == 0 {
-				nk = "K"
-			}
-			nk = nk + string(c)
-		} else if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') {
-			nk = nk + string(c)
-		} else if len(nk) > 0 {
-			nk = nk + "_"
-		}
-	}
-	return nk
-}
diff --git a/utils/models/utils.go b/utils/models/utils.go
@@ -751,3 +751,32 @@ func GetMapList(data string) map[string]string {
 	}
 	return ret
 }
+
+func PandoraKey(key string) string {
+	var nk string
+	for _, c := range key {
+		if c >= '0' && c <= '9' {
+			if len(nk) == 0 {
+				nk = "K"
+			}
+			nk = nk + string(c)
+		} else if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') {
+			nk = nk + string(c)
+		} else if len(nk) > 0 {
+			nk = nk + "_"
+		}
+	}
+	return nk
+}
+
+func DeepConvertKey(data map[string]interface{}) map[string]interface{} {
+	newData := make(map[string]interface{})
+	for k, v := range data {
+		nk := PandoraKey(k)
+		if nv, ok := v.(map[string]interface{}); ok {
+			v = DeepConvertKey(nv)
+		}
+		newData[nk] = v
+	}
+	return newData
+}
diff --git a/utils/models/utils_test.go b/utils/models/utils_test.go
@@ -614,3 +614,47 @@ func TestPickMapValue(t *testing.T) {
 	PickMapValue(m, pick, "multi", "otherword")
 	assert.NotEqual(t, exp, pick)
 }
+
+func TestPandoraKey(t *testing.T) {
+	testKeys := []string{"@timestamp", ".dot", "percent%100", "^^^^^^^^^^", "timestamp"}
+	expectKeys := []string{"timestamp", "dot", "percent_100", "", "timestamp"}
+	for idx, key := range testKeys {
+		actual := PandoraKey(key)
+		assert.Equal(t, expectKeys[idx], actual)
+	}
+}
+
+func TestDeepConvertKey(t *testing.T) {
+	testDatas := []map[string]interface{}{
+		{
+			"@timestamp": "2018-07-18T10:17:36.549054846+08:00",
+			//"timestamp":  "2018-07-19T10:17:36.549054846+08:00",
+		},
+		{
+			".dot": "dot",
+		},
+		{
+			"dot":         "dot",
+			"percent%100": 100,
+			"^^^^^^^^^^":  "mytest",
+		},
+	}
+	expectDatas := []map[string]interface{}{
+		{
+			"timestamp": "2018-07-18T10:17:36.549054846+08:00",
+		},
+		{
+			"dot": "dot",
+		},
+		{
+			"dot":         "dot",
+			"percent_100": 100,
+			"":            "mytest",
+		},
+	}
+
+	for idx, data := range testDatas {
+		actual := DeepConvertKey(data)
+		assert.Equal(t, expectDatas[idx], actual)
+	}
+}