Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change index name in the dashboards and index-pattern #4949

Merged
merged 1 commit into from
Aug 25, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di
- Added `cloud.id` and `cloud.auth` settings, for simplifying using Beats with the Elastic Cloud. {issue}4959[4959]
- Add lz4 compression support to kafka output. {pull}4977[4977]
- Add newer kafka versions to kafka output. {pull}4977[4977]
- Configure the index name when loading the dashboards and the index pattern. {pull}4949[4949]

*Auditbeat*

Expand Down
71 changes: 47 additions & 24 deletions libbeat/dashboards/es_loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,10 @@ func (loader ElasticsearchLoader) ImportIndex(file string) error {
return err
}
var indexContent common.MapStr
json.Unmarshal(reader, &indexContent)
err = json.Unmarshal(reader, &indexContent)
if err != nil {
return fmt.Errorf("fail to unmarshal index content: %v", err)
}

indexName, ok := indexContent["title"].(string)
if !ok {
Expand Down Expand Up @@ -117,7 +120,11 @@ func (loader ElasticsearchLoader) importJSONFile(fileType string, file string) e
return fmt.Errorf("Failed to read %s. Error: %s", file, err)
}
var jsonContent map[string]interface{}
json.Unmarshal(reader, &jsonContent)
err = json.Unmarshal(reader, &jsonContent)
if err != nil {
return fmt.Errorf("fail to unmarshal json file: %v", err)
}

fileBase := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))

body, err := loader.client.LoadJSON(path+"/"+fileBase, jsonContent)
Expand Down Expand Up @@ -149,10 +156,16 @@ func (loader ElasticsearchLoader) importPanelsFromDashboard(file string) (err er
}

var jsonContent record
json.Unmarshal(reader, &jsonContent)
err = json.Unmarshal(reader, &jsonContent)
if err != nil {
return fmt.Errorf("fail to unmarshal json content: %v", err)
}

var widgets []panel
json.Unmarshal([]byte(jsonContent.PanelsJSON), &widgets)
err = json.Unmarshal([]byte(jsonContent.PanelsJSON), &widgets)
if err != nil {
return fmt.Errorf("fail to unmarshal panels content: %v", err)
}

for _, widget := range widgets {
if widget.Type == "visualization" {
Expand All @@ -175,7 +188,26 @@ func (loader ElasticsearchLoader) importPanelsFromDashboard(file string) (err er

func (loader ElasticsearchLoader) importVisualization(file string) error {
loader.statusMsg("Import visualization %s", file)
if err := loader.importJSONFile("visualization", file); err != nil {
reader, err := ioutil.ReadFile(file)
if err != nil {
return err
}
var vizContent common.MapStr
err = json.Unmarshal(reader, &vizContent)
if err != nil {
return fmt.Errorf("fail to unmarshal vizualization content %s: %v", file, err)
}

if loader.config.Index != "" {
if savedObject, ok := vizContent["kibanaSavedObjectMeta"].(map[string]interface{}); ok {

vizContent["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(loader.config.Index, savedObject)
}
}

vizName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
path := "/" + loader.config.KibanaIndex + "/visualization/" + vizName
if _, err := loader.client.LoadJSON(path, vizContent); err != nil {
return err
}

Expand All @@ -192,29 +224,16 @@ func (loader ElasticsearchLoader) importSearch(file string) error {
var searchContent common.MapStr
err = json.Unmarshal(reader, &searchContent)
if err != nil {
return fmt.Errorf("Failed to unmarshal search content %s: %v", searchName, err)
return fmt.Errorf("fail to unmarshal search content %s: %v", searchName, err)
}

if loader.config.Index != "" {

// change index pattern name
if savedObject, ok := searchContent["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
if source, ok := savedObject["searchSourceJSON"].(string); ok {
var record common.MapStr
err = json.Unmarshal([]byte(source), &record)
if err != nil {
return fmt.Errorf("Failed to unmarshal searchSourceJSON from search %s: %v", searchName, err)
}

if _, ok := record["index"]; ok {
record["index"] = loader.config.Index
}
searchSourceJSON, err := json.Marshal(record)
if err != nil {
return fmt.Errorf("Failed to marshal searchSourceJSON: %v", err)
}

savedObject["searchSourceJSON"] = string(searchSourceJSON)
}

searchContent["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(loader.config.Index, savedObject)

}
}

Expand All @@ -240,7 +259,11 @@ func (loader ElasticsearchLoader) importSearchFromVisualization(file string) err
}

var jsonContent record
json.Unmarshal(reader, &jsonContent)
err = json.Unmarshal(reader, &jsonContent)
if err != nil {
return fmt.Errorf("fail to unmarshal the search content: %v", err)
}

id := jsonContent.SavedSearchID
if len(id) == 0 {
// no search used
Expand Down
25 changes: 20 additions & 5 deletions libbeat/dashboards/kibana_loader.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package dashboards

import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/url"
Expand Down Expand Up @@ -48,12 +48,20 @@ func (loader KibanaLoader) ImportIndex(file string) error {
params.Set("force", "true") //overwrite the existing dashboards

// read json file
content, err := ioutil.ReadFile(file)
reader, err := ioutil.ReadFile(file)
if err != nil {
return fmt.Errorf("fail to read index-pattern: %v", err)
}

return loader.client.ImportJSON(importAPI, params, bytes.NewBuffer(content))
var indexContent common.MapStr
err = json.Unmarshal(reader, &indexContent)
if err != nil {
return fmt.Errorf("fail to unmarshal the index content: %v", err)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

s/fail/failed/ -> not sure which one we shoud use, but we should be consistent :-)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I always using fail instead of failed. Not sure what is more widely used.

}

indexContent = ReplaceIndexInIndexPattern(loader.config.Index, indexContent)

return loader.client.ImportJSON(importAPI, params, indexContent)
}

func (loader KibanaLoader) ImportDashboard(file string) error {
Expand All @@ -62,12 +70,19 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
params.Add("exclude", "index-pattern") //don't import the index pattern from the dashboards

// read json file
content, err := ioutil.ReadFile(file)
reader, err := ioutil.ReadFile(file)
if err != nil {
return fmt.Errorf("fail to read index-pattern: %v", err)
}
var content common.MapStr
err = json.Unmarshal(reader, &content)
if err != nil {
return fmt.Errorf("fail to unmarshal the index content: %v", err)
}

content = ReplaceIndexInDashboardObject(loader.config.Index, content)

return loader.client.ImportJSON(importAPI, params, bytes.NewBuffer(content))
return loader.client.ImportJSON(importAPI, params, content)
}

func (loader KibanaLoader) Close() error {
Expand Down
105 changes: 105 additions & 0 deletions libbeat/dashboards/modify_json.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package dashboards

import (
"encoding/json"
"fmt"

"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/logp"
)

type JSONObjectAttribute struct {
Description string `json:"description"`
KibanaSavedObjectMeta map[string]interface{} `json:"kibanaSavedObjectMeta"`
Title string `json:"title"`
Type string `json:"type"`
}

type JSONObject struct {
Attributes JSONObjectAttribute `json:"attributes"`
}

type JSONFormat struct {
Objects []JSONObject `json:"objects"`
}

func ReplaceIndexInIndexPattern(index string, content common.MapStr) common.MapStr {

if index != "" {
// change index pattern name
if objects, ok := content["objects"].([]interface{}); ok {
for i, object := range objects {
if objectMap, ok := object.(map[string]interface{}); ok {
objectMap["id"] = index

if attributes, ok := objectMap["attributes"].(map[string]interface{}); ok {
attributes["title"] = index
objectMap["attributes"] = attributes
}
objects[i] = objectMap
}
}
content["objects"] = objects
}
}

return content
}

func replaceIndexInSearchObject(index string, savedObject string) (string, error) {

var record common.MapStr
err := json.Unmarshal([]byte(savedObject), &record)
if err != nil {
return "", fmt.Errorf("fail to unmarshal searchSourceJSON from search : %v", err)
}

if _, ok := record["index"]; ok {
record["index"] = index
}
searchSourceJSON, err := json.Marshal(record)
if err != nil {
return "", fmt.Errorf("fail to marshal searchSourceJSON: %v", err)
}

return string(searchSourceJSON), nil
}

func ReplaceIndexInSavedObject(index string, kibanaSavedObject map[string]interface{}) map[string]interface{} {

if searchSourceJSON, ok := kibanaSavedObject["searchSourceJSON"].(string); ok {
searchSourceJSON, err := replaceIndexInSearchObject(index, searchSourceJSON)
if err != nil {
logp.Err("Fail to replace searchSourceJSON: %v", err)
return kibanaSavedObject
}
kibanaSavedObject["searchSourceJSON"] = searchSourceJSON
}

return kibanaSavedObject
}

func ReplaceIndexInDashboardObject(index string, content common.MapStr) common.MapStr {

if index == "" {
return content
}
if objects, ok := content["objects"].([]interface{}); ok {
for i, object := range objects {
if objectMap, ok := object.(map[string]interface{}); ok {
if attributes, ok := objectMap["attributes"].(map[string]interface{}); ok {

if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok {

attributes["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(index, kibanaSavedObject)
}

objectMap["attributes"] = attributes
}
objects[i] = objectMap
}
}
content["objects"] = objects
}
return content
}
11 changes: 9 additions & 2 deletions libbeat/setup/kibana/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,15 @@ func (client *Client) SetVersion() error {

func (client *Client) GetVersion() string { return client.version }

func (client *Client) ImportJSON(url string, params url.Values, body io.Reader) error {
statusCode, response, err := client.Connection.Request("POST", url, params, body)
func (client *Client) ImportJSON(url string, params url.Values, jsonBody map[string]interface{}) error {

body, err := json.Marshal(jsonBody)
if err != nil {
logp.Err("Failed to json encode body (%v): %#v", err, jsonBody)
return fmt.Errorf("fail to marshal the json content: %v", err)
}

statusCode, response, err := client.Connection.Request("POST", url, params, bytes.NewBuffer(body))
if err != nil {
return fmt.Errorf("%v. Response: %s", err, truncateString(response))
}
Expand Down