From 0bc31863c65dc9722b0ea70c9cc4f5a7c1f9e45f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 21 Nov 2017 17:19:39 +0100 Subject: [PATCH] Support dashboard loading without Elasticsearch (#5653) * libbeat: support dashboard loading without Elasticsearch A new boolean option for dashboards is introduced named alwas_kibana. If it is true, Elasticsearch version check is skipped and dashboards are loaded using Kibana API. It should be used when no Elasticsearch output is configured. * additional docs and changelog fix --- CHANGELOG.asciidoc | 2 ++ auditbeat/auditbeat.reference.yml | 3 +++ filebeat/filebeat.reference.yml | 3 +++ heartbeat/heartbeat.reference.yml | 3 +++ libbeat/_meta/config.reference.yml | 3 +++ libbeat/dashboards/config.go | 4 +++- libbeat/dashboards/dashboards.go | 20 +++++++++++++++----- libbeat/docs/dashboardsconfig.asciidoc | 6 ++++++ metricbeat/metricbeat.reference.yml | 3 +++ packetbeat/packetbeat.reference.yml | 3 +++ winlogbeat/winlogbeat.reference.yml | 3 +++ 11 files changed, 47 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 0ba7079a5f0..6267b8c2684 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -14,6 +14,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di *Affecting all Beats* +- Support dashboard loading without Elasticseach {pull}5653[5653] + *Auditbeat* - Changed `audit.file.path` to be a multi-field so that path is searchable. {pull}5625[5625] diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index dfad07b2638..999e6676bf3 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -722,6 +722,9 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version +#always_kibana: false + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index c10cac71b0e..b2ef6248b47 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1150,6 +1150,9 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version +#always_kibana: false + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 5551500da64..328a056551e 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -842,6 +842,9 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version +#always_kibana: false + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index e1bebc42897..a79c333b141 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -628,6 +628,9 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version +#always_kibana: false + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch diff --git a/libbeat/dashboards/config.go b/libbeat/dashboards/config.go index ab1dfc13226..5c6db62c15b 100644 --- a/libbeat/dashboards/config.go +++ b/libbeat/dashboards/config.go @@ -10,10 +10,12 @@ type Config struct { URL string `config:"url"` OnlyDashboards bool `config:"only_dashboards"` OnlyIndex bool `config:"only_index"` + AlwaysKibana bool `config:"always_kibana"` } var defaultConfig = Config{ - KibanaIndex: ".kibana", + KibanaIndex: ".kibana", + AlwaysKibana: false, } var ( defaultDirectory = "kibana" diff --git a/libbeat/dashboards/dashboards.go b/libbeat/dashboards/dashboards.go index 0927e406c06..ab34d85f527 100644 --- a/libbeat/dashboards/dashboards.go +++ b/libbeat/dashboards/dashboards.go @@ -29,6 +29,14 @@ func ImportDashboards(beatName, hostname, homePath string, return err } + if kibanaConfig == nil { + kibanaConfig = common.NewConfig() + } + + if esConfig == nil && dashConfig.AlwaysKibana { + return setupAndImportDashboardsViaKibana(hostname, kibanaConfig, &dashConfig, msgOutputter) + } + esLoader, err := NewElasticsearchLoader(esConfig, &dashConfig, msgOutputter) if err != nil { return fmt.Errorf("fail to create the Elasticsearch loader: %v", err) @@ -48,10 +56,6 @@ func ImportDashboards(beatName, hostname, homePath string, logp.Info("For Elasticsearch version >= 6.0.0, the Kibana dashboards need to be imported via the Kibana API.") - if kibanaConfig == nil { - kibanaConfig = common.NewConfig() - } - // In Cloud, the Kibana URL is different than the Elasticsearch URL, // but the credentials are the same. // So, by default, use same credentials for connecting to Kibana as to Elasticsearch @@ -62,7 +66,13 @@ func ImportDashboards(beatName, hostname, homePath string, kibanaConfig.SetString("password", -1, esLoader.client.Password) } - kibanaLoader, err := NewKibanaLoader(kibanaConfig, &dashConfig, hostname, msgOutputter) + return setupAndImportDashboardsViaKibana(hostname, kibanaConfig, &dashConfig, msgOutputter) +} + +func setupAndImportDashboardsViaKibana(hostname string, kibanaConfig *common.Config, + dashboardsConfig *Config, msgOutputter MessageOutputter) error { + + kibanaLoader, err := NewKibanaLoader(kibanaConfig, dashboardsConfig, hostname, msgOutputter) if err != nil { return fmt.Errorf("fail to create the Kibana loader: %v", err) } diff --git a/libbeat/docs/dashboardsconfig.asciidoc b/libbeat/docs/dashboardsconfig.asciidoc index 4c59b060191..03a24c41c5d 100644 --- a/libbeat/docs/dashboardsconfig.asciidoc +++ b/libbeat/docs/dashboardsconfig.asciidoc @@ -91,3 +91,9 @@ is `".kibana"` The Elasticsearch index name. This setting overwrites the index name defined in the dashboards and index pattern. Example: `"testbeat-*"` + +[float] +==== `setup.dashboards.always_kibana` + +Force loading of dashboards using the Kibana API without querying Elasticsearch for the version +The default is `false`. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index a245f7b3d7d..f67466ce215 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1109,6 +1109,9 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version +#always_kibana: false + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 29bc7a465fb..63e0a94b0b6 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1096,6 +1096,9 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version +#always_kibana: false + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 1ff92640cc8..53ea4686894 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -657,6 +657,9 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version +#always_kibana: false + #============================== Template ===================================== # A template is used to set the mapping in Elasticsearch