#6066: Extract json deDot feature from metricbeat/http module to libbeat/common

libbeat/common/event.go

@@ -256,3 +256,24 @@ func (f Float) MarshalJSON() ([]byte, error) {
 func DeDot(s string) string {
 	return strings.Replace(s, ".", "_", -1)
 }
+
+// DeDotJSON replaces in keys all . with _
+// This helps when sending data to Elasticsearch to prevent object and key collisions.
+func DeDotJSON(json interface{}) interface{} {
+	switch json.(type) {
+	case map[string]interface{}:
+		result := map[string]interface{}{}
+		for key, value := range json.(map[string]interface{}) {
+			result[DeDot(key)] = DeDotJSON(value)
+		}
+		return result
+	case []interface{}:
+		result := make([]interface{}, len(json.([]interface{})))
+		for i, value := range json.([]interface{}) {
+			result[i] = DeDotJSON(value)
+		}
+		return result
+	default:
+		return json
+	}
+}

libbeat/common/event_test.go

@@ -7,6 +7,9 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
+	"io/ioutil"
+	"path/filepath"
+
 	"github.com/elastic/beats/libbeat/logp"
 )
 
@@ -379,3 +382,49 @@ func BenchmarkConvertToGenericEventStringPointer(b *testing.B) {
 		ConvertToGenericEvent(MapStr{"key": &val})
 	}
 }
+
+func TestDeDotJsonMap(t *testing.T) {
+	var actualJSONBody map[string]interface{}
+	var expectedJSONBody map[string]interface{}
+
+	absPath, err := filepath.Abs("./testdata")
+	assert.NotNil(t, absPath)
+	assert.Nil(t, err)
+
+	actualJSONResponse, err := ioutil.ReadFile(absPath + "/json_map_with_dots.json")
+	assert.Nil(t, err)
+	err = json.Unmarshal(actualJSONResponse, &actualJSONBody)
+	assert.Nil(t, err)
+
+	dedottedJSONResponse, err := ioutil.ReadFile(absPath + "/json_map_dedot.json")
+	assert.Nil(t, err)
+	err = json.Unmarshal(dedottedJSONResponse, &expectedJSONBody)
+	assert.Nil(t, err)
+
+	actualJSONBody = DeDotJSON(actualJSONBody).(map[string]interface{})
+
+	assert.Equal(t, expectedJSONBody, actualJSONBody)
+}
+
+func TestDeDotJsonArray(t *testing.T) {
+	var actualJSONBody []interface{}
+	var expectedJSONBody []interface{}
+
+	absPath, err := filepath.Abs("./testdata")
+	assert.NotNil(t, absPath)
+	assert.Nil(t, err)
+
+	actualJSONResponse, err := ioutil.ReadFile(absPath + "/json_array_with_dots.json")
+	assert.Nil(t, err)
+	err = json.Unmarshal(actualJSONResponse, &actualJSONBody)
+	assert.Nil(t, err)
+
+	dedottedJSONResponse, err := ioutil.ReadFile(absPath + "/json_array_dedot.json")
+	assert.Nil(t, err)
+	err = json.Unmarshal(dedottedJSONResponse, &expectedJSONBody)
+	assert.Nil(t, err)
+
+	actualJSONBody = DeDotJSON(actualJSONBody).([]interface{})
+
+	assert.Equal(t, expectedJSONBody, actualJSONBody)
+}

libbeat/common/testdata/json_array_dedot.json

@@ -0,0 +1,5 @@
+[
+    {"key_with_dot_1":"value1_1"},
+    {"key_without_dot_2":"value1_2"},
+    {"key_with_multiple_dots_3": {"key_with_dot_2":"value2_1"}}
+]

libbeat/common/testdata/json_array_with_dots.json

@@ -0,0 +1,5 @@
+[
+    {"key_with_dot.1":"value1_1"},
+    {"key_without_dot_2":"value1_2"},
+    {"key_with_multiple.dots.3": {"key_with_dot.2":"value2_1"}}
+]

metricbeat/module/http/json/_meta/test/json_response_dedot.json → libbeat/common/testdata/json_map_dedot.json

@@ -3,8 +3,8 @@
         "key_with_dot_l2": 1,
         "key_with_multiple_dots_l2": 2,
         "key_without_dot_l2": {
-            "key_with_dot_l2": 3,
-            "key_with_multiple_dots_l2": 4
+            "key_with_dot_l3": 3,
+            "key_with_multiple_dots_l3": 4
         }
     }
 }

metricbeat/module/http/json/_meta/test/json_response_with_dots.json → libbeat/common/testdata/json_map_with_dots.json

@@ -1,10 +1,10 @@
 {
     "key_without_dot_l1": {
-        "key_with_dot_l2": 1,
+        "key_with_dot.l2": 1,
         "key.with.multiple.dots_l2": 2,
         "key_without_dot_l2": {
-            "key_with_dot.l2": 3,
-            "key.with.multiple.dots_l2": 4
+            "key_with_dot.l3": 3,
+            "key.with.multiple.dots_l3": 4
         }
     }
 }

metricbeat/module/http/json/json.go

@@ -118,7 +118,7 @@ func (m *MetricSet) Fetch() (common.MapStr, error) {
 	}
 
 	if m.deDotEnabled {
-		event = replaceDots(jsonBody).(map[string]interface{})
+		event = common.DeDotJSON(jsonBody).(map[string]interface{})
 	} else {
 		event = jsonBody
 	}
@@ -162,16 +162,3 @@ func (m *MetricSet) getHeaders(header http.Header) map[string]string {
 	}
 	return headers
 }
-
-func replaceDots(data interface{}) interface{} {
-	switch data.(type) {
-	case map[string]interface{}:
-		result := map[string]interface{}{}
-		for key, value := range data.(map[string]interface{}) {
-			result[common.DeDot(key)] = replaceDots(value)
-		}
-		return result
-	default:
-		return data
-	}
-}