From 688882e442b06b9d6a9b8461821c99167c158a6c Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Fri, 25 Aug 2017 15:00:51 +0200 Subject: [PATCH] Make the index name configurable when loading the 6.x and 5.x dashboards (#4949) (#5018) (cherry picked from commit 36932d6e2af33963b287543891c9de8aa7250fe2) --- CHANGELOG.asciidoc | 1 + libbeat/dashboards/es_loader.go | 71 ++++++++++++------- libbeat/dashboards/kibana_loader.go | 25 +++++-- libbeat/dashboards/modify_json.go | 105 ++++++++++++++++++++++++++++ libbeat/setup/kibana/client.go | 11 ++- 5 files changed, 182 insertions(+), 31 deletions(-) create mode 100644 libbeat/dashboards/modify_json.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 047837de842..ebcde6349eb 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -73,6 +73,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di - Added `cloud.id` and `cloud.auth` settings, for simplifying using Beats with the Elastic Cloud. {issue}4959[4959] - Add lz4 compression support to kafka output. {pull}4977[4977] - Add newer kafka versions to kafka output. {pull}4977[4977] +- Configure the index name when loading the dashboards and the index pattern. {pull}4949[4949] *Auditbeat* diff --git a/libbeat/dashboards/es_loader.go b/libbeat/dashboards/es_loader.go index f14394662e0..8c2784138fc 100644 --- a/libbeat/dashboards/es_loader.go +++ b/libbeat/dashboards/es_loader.go @@ -87,7 +87,10 @@ func (loader ElasticsearchLoader) ImportIndex(file string) error { return err } var indexContent common.MapStr - json.Unmarshal(reader, &indexContent) + err = json.Unmarshal(reader, &indexContent) + if err != nil { + return fmt.Errorf("fail to unmarshal index content: %v", err) + } indexName, ok := indexContent["title"].(string) if !ok { @@ -117,7 +120,11 @@ func (loader ElasticsearchLoader) importJSONFile(fileType string, file string) e return fmt.Errorf("Failed to read %s. Error: %s", file, err) } var jsonContent map[string]interface{} - json.Unmarshal(reader, &jsonContent) + err = json.Unmarshal(reader, &jsonContent) + if err != nil { + return fmt.Errorf("fail to unmarshal json file: %v", err) + } + fileBase := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file)) body, err := loader.client.LoadJSON(path+"/"+fileBase, jsonContent) @@ -149,10 +156,16 @@ func (loader ElasticsearchLoader) importPanelsFromDashboard(file string) (err er } var jsonContent record - json.Unmarshal(reader, &jsonContent) + err = json.Unmarshal(reader, &jsonContent) + if err != nil { + return fmt.Errorf("fail to unmarshal json content: %v", err) + } var widgets []panel - json.Unmarshal([]byte(jsonContent.PanelsJSON), &widgets) + err = json.Unmarshal([]byte(jsonContent.PanelsJSON), &widgets) + if err != nil { + return fmt.Errorf("fail to unmarshal panels content: %v", err) + } for _, widget := range widgets { if widget.Type == "visualization" { @@ -175,7 +188,26 @@ func (loader ElasticsearchLoader) importPanelsFromDashboard(file string) (err er func (loader ElasticsearchLoader) importVisualization(file string) error { loader.statusMsg("Import visualization %s", file) - if err := loader.importJSONFile("visualization", file); err != nil { + reader, err := ioutil.ReadFile(file) + if err != nil { + return err + } + var vizContent common.MapStr + err = json.Unmarshal(reader, &vizContent) + if err != nil { + return fmt.Errorf("fail to unmarshal vizualization content %s: %v", file, err) + } + + if loader.config.Index != "" { + if savedObject, ok := vizContent["kibanaSavedObjectMeta"].(map[string]interface{}); ok { + + vizContent["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(loader.config.Index, savedObject) + } + } + + vizName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file)) + path := "/" + loader.config.KibanaIndex + "/visualization/" + vizName + if _, err := loader.client.LoadJSON(path, vizContent); err != nil { return err } @@ -192,29 +224,16 @@ func (loader ElasticsearchLoader) importSearch(file string) error { var searchContent common.MapStr err = json.Unmarshal(reader, &searchContent) if err != nil { - return fmt.Errorf("Failed to unmarshal search content %s: %v", searchName, err) + return fmt.Errorf("fail to unmarshal search content %s: %v", searchName, err) } if loader.config.Index != "" { + // change index pattern name if savedObject, ok := searchContent["kibanaSavedObjectMeta"].(map[string]interface{}); ok { - if source, ok := savedObject["searchSourceJSON"].(string); ok { - var record common.MapStr - err = json.Unmarshal([]byte(source), &record) - if err != nil { - return fmt.Errorf("Failed to unmarshal searchSourceJSON from search %s: %v", searchName, err) - } - - if _, ok := record["index"]; ok { - record["index"] = loader.config.Index - } - searchSourceJSON, err := json.Marshal(record) - if err != nil { - return fmt.Errorf("Failed to marshal searchSourceJSON: %v", err) - } - - savedObject["searchSourceJSON"] = string(searchSourceJSON) - } + + searchContent["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(loader.config.Index, savedObject) + } } @@ -240,7 +259,11 @@ func (loader ElasticsearchLoader) importSearchFromVisualization(file string) err } var jsonContent record - json.Unmarshal(reader, &jsonContent) + err = json.Unmarshal(reader, &jsonContent) + if err != nil { + return fmt.Errorf("fail to unmarshal the search content: %v", err) + } + id := jsonContent.SavedSearchID if len(id) == 0 { // no search used diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index 882a606d28f..ba58a931b7e 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -1,7 +1,7 @@ package dashboards import ( - "bytes" + "encoding/json" "fmt" "io/ioutil" "net/url" @@ -48,12 +48,20 @@ func (loader KibanaLoader) ImportIndex(file string) error { params.Set("force", "true") //overwrite the existing dashboards // read json file - content, err := ioutil.ReadFile(file) + reader, err := ioutil.ReadFile(file) if err != nil { return fmt.Errorf("fail to read index-pattern: %v", err) } - return loader.client.ImportJSON(importAPI, params, bytes.NewBuffer(content)) + var indexContent common.MapStr + err = json.Unmarshal(reader, &indexContent) + if err != nil { + return fmt.Errorf("fail to unmarshal the index content: %v", err) + } + + indexContent = ReplaceIndexInIndexPattern(loader.config.Index, indexContent) + + return loader.client.ImportJSON(importAPI, params, indexContent) } func (loader KibanaLoader) ImportDashboard(file string) error { @@ -62,12 +70,19 @@ func (loader KibanaLoader) ImportDashboard(file string) error { params.Add("exclude", "index-pattern") //don't import the index pattern from the dashboards // read json file - content, err := ioutil.ReadFile(file) + reader, err := ioutil.ReadFile(file) if err != nil { return fmt.Errorf("fail to read index-pattern: %v", err) } + var content common.MapStr + err = json.Unmarshal(reader, &content) + if err != nil { + return fmt.Errorf("fail to unmarshal the index content: %v", err) + } + + content = ReplaceIndexInDashboardObject(loader.config.Index, content) - return loader.client.ImportJSON(importAPI, params, bytes.NewBuffer(content)) + return loader.client.ImportJSON(importAPI, params, content) } func (loader KibanaLoader) Close() error { diff --git a/libbeat/dashboards/modify_json.go b/libbeat/dashboards/modify_json.go new file mode 100644 index 00000000000..2a219bfdfeb --- /dev/null +++ b/libbeat/dashboards/modify_json.go @@ -0,0 +1,105 @@ +package dashboards + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" +) + +type JSONObjectAttribute struct { + Description string `json:"description"` + KibanaSavedObjectMeta map[string]interface{} `json:"kibanaSavedObjectMeta"` + Title string `json:"title"` + Type string `json:"type"` +} + +type JSONObject struct { + Attributes JSONObjectAttribute `json:"attributes"` +} + +type JSONFormat struct { + Objects []JSONObject `json:"objects"` +} + +func ReplaceIndexInIndexPattern(index string, content common.MapStr) common.MapStr { + + if index != "" { + // change index pattern name + if objects, ok := content["objects"].([]interface{}); ok { + for i, object := range objects { + if objectMap, ok := object.(map[string]interface{}); ok { + objectMap["id"] = index + + if attributes, ok := objectMap["attributes"].(map[string]interface{}); ok { + attributes["title"] = index + objectMap["attributes"] = attributes + } + objects[i] = objectMap + } + } + content["objects"] = objects + } + } + + return content +} + +func replaceIndexInSearchObject(index string, savedObject string) (string, error) { + + var record common.MapStr + err := json.Unmarshal([]byte(savedObject), &record) + if err != nil { + return "", fmt.Errorf("fail to unmarshal searchSourceJSON from search : %v", err) + } + + if _, ok := record["index"]; ok { + record["index"] = index + } + searchSourceJSON, err := json.Marshal(record) + if err != nil { + return "", fmt.Errorf("fail to marshal searchSourceJSON: %v", err) + } + + return string(searchSourceJSON), nil +} + +func ReplaceIndexInSavedObject(index string, kibanaSavedObject map[string]interface{}) map[string]interface{} { + + if searchSourceJSON, ok := kibanaSavedObject["searchSourceJSON"].(string); ok { + searchSourceJSON, err := replaceIndexInSearchObject(index, searchSourceJSON) + if err != nil { + logp.Err("Fail to replace searchSourceJSON: %v", err) + return kibanaSavedObject + } + kibanaSavedObject["searchSourceJSON"] = searchSourceJSON + } + + return kibanaSavedObject +} + +func ReplaceIndexInDashboardObject(index string, content common.MapStr) common.MapStr { + + if index == "" { + return content + } + if objects, ok := content["objects"].([]interface{}); ok { + for i, object := range objects { + if objectMap, ok := object.(map[string]interface{}); ok { + if attributes, ok := objectMap["attributes"].(map[string]interface{}); ok { + + if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok { + + attributes["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(index, kibanaSavedObject) + } + + objectMap["attributes"] = attributes + } + objects[i] = objectMap + } + } + content["objects"] = objects + } + return content +} diff --git a/libbeat/setup/kibana/client.go b/libbeat/setup/kibana/client.go index 85498206326..07baa9efe5a 100644 --- a/libbeat/setup/kibana/client.go +++ b/libbeat/setup/kibana/client.go @@ -196,8 +196,15 @@ func (client *Client) SetVersion() error { func (client *Client) GetVersion() string { return client.version } -func (client *Client) ImportJSON(url string, params url.Values, body io.Reader) error { - statusCode, response, err := client.Connection.Request("POST", url, params, body) +func (client *Client) ImportJSON(url string, params url.Values, jsonBody map[string]interface{}) error { + + body, err := json.Marshal(jsonBody) + if err != nil { + logp.Err("Failed to json encode body (%v): %#v", err, jsonBody) + return fmt.Errorf("fail to marshal the json content: %v", err) + } + + statusCode, response, err := client.Connection.Request("POST", url, params, bytes.NewBuffer(body)) if err != nil { return fmt.Errorf("%v. Response: %s", err, truncateString(response)) }