Make the index name configurable when loading the 6.x and 5.x dashboa…

…rds (elastic#4949) (elastic#5018)

(cherry picked from commit 36932d6)

CHANGELOG.asciidoc

@@ -73,6 +73,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di
 - Added `cloud.id` and `cloud.auth` settings, for simplifying using Beats with the Elastic Cloud. {issue}4959[4959]
 - Add lz4 compression support to kafka output. {pull}4977[4977]
 - Add newer kafka versions to kafka output. {pull}4977[4977]
+- Configure the index name when loading the dashboards and the index pattern. {pull}4949[4949]
 
 *Auditbeat*
 

libbeat/dashboards/es_loader.go

@@ -87,7 +87,10 @@ func (loader ElasticsearchLoader) ImportIndex(file string) error {
 		return err
 	}
 	var indexContent common.MapStr
-	json.Unmarshal(reader, &indexContent)
+	err = json.Unmarshal(reader, &indexContent)
+	if err != nil {
+		return fmt.Errorf("fail to unmarshal index content: %v", err)
+	}
 
 	indexName, ok := indexContent["title"].(string)
 	if !ok {
@@ -117,7 +120,11 @@ func (loader ElasticsearchLoader) importJSONFile(fileType string, file string) e
 		return fmt.Errorf("Failed to read %s. Error: %s", file, err)
 	}
 	var jsonContent map[string]interface{}
-	json.Unmarshal(reader, &jsonContent)
+	err = json.Unmarshal(reader, &jsonContent)
+	if err != nil {
+		return fmt.Errorf("fail to unmarshal json file: %v", err)
+	}
+
 	fileBase := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
 
 	body, err := loader.client.LoadJSON(path+"/"+fileBase, jsonContent)
@@ -149,10 +156,16 @@ func (loader ElasticsearchLoader) importPanelsFromDashboard(file string) (err er
 	}
 
 	var jsonContent record
-	json.Unmarshal(reader, &jsonContent)
+	err = json.Unmarshal(reader, &jsonContent)
+	if err != nil {
+		return fmt.Errorf("fail to unmarshal json content: %v", err)
+	}
 
 	var widgets []panel
-	json.Unmarshal([]byte(jsonContent.PanelsJSON), &widgets)
+	err = json.Unmarshal([]byte(jsonContent.PanelsJSON), &widgets)
+	if err != nil {
+		return fmt.Errorf("fail to unmarshal panels content: %v", err)
+	}
 
 	for _, widget := range widgets {
 		if widget.Type == "visualization" {
@@ -175,7 +188,26 @@ func (loader ElasticsearchLoader) importPanelsFromDashboard(file string) (err er
 
 func (loader ElasticsearchLoader) importVisualization(file string) error {
 	loader.statusMsg("Import visualization %s", file)
-	if err := loader.importJSONFile("visualization", file); err != nil {
+	reader, err := ioutil.ReadFile(file)
+	if err != nil {
+		return err
+	}
+	var vizContent common.MapStr
+	err = json.Unmarshal(reader, &vizContent)
+	if err != nil {
+		return fmt.Errorf("fail to unmarshal vizualization content %s: %v", file, err)
+	}
+
+	if loader.config.Index != "" {
+		if savedObject, ok := vizContent["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
+
+			vizContent["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(loader.config.Index, savedObject)
+		}
+	}
+
+	vizName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
+	path := "/" + loader.config.KibanaIndex + "/visualization/" + vizName
+	if _, err := loader.client.LoadJSON(path, vizContent); err != nil {
 		return err
 	}
 
@@ -192,29 +224,16 @@ func (loader ElasticsearchLoader) importSearch(file string) error {
 	var searchContent common.MapStr
 	err = json.Unmarshal(reader, &searchContent)
 	if err != nil {
-		return fmt.Errorf("Failed to unmarshal search content %s: %v", searchName, err)
+		return fmt.Errorf("fail to unmarshal search content %s: %v", searchName, err)
 	}
 
 	if loader.config.Index != "" {
+
 		// change index pattern name
 		if savedObject, ok := searchContent["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
-			if source, ok := savedObject["searchSourceJSON"].(string); ok {
-				var record common.MapStr
-				err = json.Unmarshal([]byte(source), &record)
-				if err != nil {
-					return fmt.Errorf("Failed to unmarshal searchSourceJSON from search %s: %v", searchName, err)
-				}
-
-				if _, ok := record["index"]; ok {
-					record["index"] = loader.config.Index
-				}
-				searchSourceJSON, err := json.Marshal(record)
-				if err != nil {
-					return fmt.Errorf("Failed to marshal searchSourceJSON: %v", err)
-				}
-
-				savedObject["searchSourceJSON"] = string(searchSourceJSON)
-			}
+
+			searchContent["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(loader.config.Index, savedObject)
+
 		}
 	}
 
@@ -240,7 +259,11 @@ func (loader ElasticsearchLoader) importSearchFromVisualization(file string) err
 	}
 
 	var jsonContent record
-	json.Unmarshal(reader, &jsonContent)
+	err = json.Unmarshal(reader, &jsonContent)
+	if err != nil {
+		return fmt.Errorf("fail to unmarshal the search content: %v", err)
+	}
+
 	id := jsonContent.SavedSearchID
 	if len(id) == 0 {
 		// no search used

libbeat/dashboards/kibana_loader.go

@@ -1,7 +1,7 @@
 package dashboards
 
 import (
-	"bytes"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/url"
@@ -48,12 +48,20 @@ func (loader KibanaLoader) ImportIndex(file string) error {
 	params.Set("force", "true") //overwrite the existing dashboards
 
 	// read json file
-	content, err := ioutil.ReadFile(file)
+	reader, err := ioutil.ReadFile(file)
 	if err != nil {
 		return fmt.Errorf("fail to read index-pattern: %v", err)
 	}
 
-	return loader.client.ImportJSON(importAPI, params, bytes.NewBuffer(content))
+	var indexContent common.MapStr
+	err = json.Unmarshal(reader, &indexContent)
+	if err != nil {
+		return fmt.Errorf("fail to unmarshal the index content: %v", err)
+	}
+
+	indexContent = ReplaceIndexInIndexPattern(loader.config.Index, indexContent)
+
+	return loader.client.ImportJSON(importAPI, params, indexContent)
 }
 
 func (loader KibanaLoader) ImportDashboard(file string) error {
@@ -62,12 +70,19 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
 	params.Add("exclude", "index-pattern") //don't import the index pattern from the dashboards
 
 	// read json file
-	content, err := ioutil.ReadFile(file)
+	reader, err := ioutil.ReadFile(file)
 	if err != nil {
 		return fmt.Errorf("fail to read index-pattern: %v", err)
 	}
+	var content common.MapStr
+	err = json.Unmarshal(reader, &content)
+	if err != nil {
+		return fmt.Errorf("fail to unmarshal the index content: %v", err)
+	}
+
+	content = ReplaceIndexInDashboardObject(loader.config.Index, content)
 
-	return loader.client.ImportJSON(importAPI, params, bytes.NewBuffer(content))
+	return loader.client.ImportJSON(importAPI, params, content)
 }
 
 func (loader KibanaLoader) Close() error {

libbeat/dashboards/modify_json.go

@@ -0,0 +1,105 @@
+package dashboards
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/logp"
+)
+
+type JSONObjectAttribute struct {
+	Description           string                 `json:"description"`
+	KibanaSavedObjectMeta map[string]interface{} `json:"kibanaSavedObjectMeta"`
+	Title                 string                 `json:"title"`
+	Type                  string                 `json:"type"`
+}
+
+type JSONObject struct {
+	Attributes JSONObjectAttribute `json:"attributes"`
+}
+
+type JSONFormat struct {
+	Objects []JSONObject `json:"objects"`
+}
+
+func ReplaceIndexInIndexPattern(index string, content common.MapStr) common.MapStr {
+
+	if index != "" {
+		// change index pattern name
+		if objects, ok := content["objects"].([]interface{}); ok {
+			for i, object := range objects {
+				if objectMap, ok := object.(map[string]interface{}); ok {
+					objectMap["id"] = index
+
+					if attributes, ok := objectMap["attributes"].(map[string]interface{}); ok {
+						attributes["title"] = index
+						objectMap["attributes"] = attributes
+					}
+					objects[i] = objectMap
+				}
+			}
+			content["objects"] = objects
+		}
+	}
+
+	return content
+}
+
+func replaceIndexInSearchObject(index string, savedObject string) (string, error) {
+
+	var record common.MapStr
+	err := json.Unmarshal([]byte(savedObject), &record)
+	if err != nil {
+		return "", fmt.Errorf("fail to unmarshal searchSourceJSON from search : %v", err)
+	}
+
+	if _, ok := record["index"]; ok {
+		record["index"] = index
+	}
+	searchSourceJSON, err := json.Marshal(record)
+	if err != nil {
+		return "", fmt.Errorf("fail to marshal searchSourceJSON: %v", err)
+	}
+
+	return string(searchSourceJSON), nil
+}
+
+func ReplaceIndexInSavedObject(index string, kibanaSavedObject map[string]interface{}) map[string]interface{} {
+
+	if searchSourceJSON, ok := kibanaSavedObject["searchSourceJSON"].(string); ok {
+		searchSourceJSON, err := replaceIndexInSearchObject(index, searchSourceJSON)
+		if err != nil {
+			logp.Err("Fail to replace searchSourceJSON: %v", err)
+			return kibanaSavedObject
+		}
+		kibanaSavedObject["searchSourceJSON"] = searchSourceJSON
+	}
+
+	return kibanaSavedObject
+}
+
+func ReplaceIndexInDashboardObject(index string, content common.MapStr) common.MapStr {
+
+	if index == "" {
+		return content
+	}
+	if objects, ok := content["objects"].([]interface{}); ok {
+		for i, object := range objects {
+			if objectMap, ok := object.(map[string]interface{}); ok {
+				if attributes, ok := objectMap["attributes"].(map[string]interface{}); ok {
+
+					if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
+
+						attributes["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(index, kibanaSavedObject)
+					}
+
+					objectMap["attributes"] = attributes
+				}
+				objects[i] = objectMap
+			}
+		}
+		content["objects"] = objects
+	}
+	return content
+}

libbeat/setup/kibana/client.go

@@ -196,8 +196,15 @@ func (client *Client) SetVersion() error {
 
 func (client *Client) GetVersion() string { return client.version }
 
-func (client *Client) ImportJSON(url string, params url.Values, body io.Reader) error {
-	statusCode, response, err := client.Connection.Request("POST", url, params, body)
+func (client *Client) ImportJSON(url string, params url.Values, jsonBody map[string]interface{}) error {
+
+	body, err := json.Marshal(jsonBody)
+	if err != nil {
+		logp.Err("Failed to json encode body (%v): %#v", err, jsonBody)
+		return fmt.Errorf("fail to marshal the json content: %v", err)
+	}
+
+	statusCode, response, err := client.Connection.Request("POST", url, params, bytes.NewBuffer(body))
 	if err != nil {
 		return fmt.Errorf("%v. Response: %s", err, truncateString(response))
 	}