Support dashboard loading without Elasticsearch (#5653)

* libbeat: support dashboard loading without Elasticsearch

A new boolean option for dashboards is introduced named alwas_kibana.
If it is true, Elasticsearch version check is skipped and dashboards
are loaded using Kibana API. It should be used when no Elasticsearch
output is configured.

* additional docs and changelog fix

CHANGELOG.asciidoc

@@ -14,6 +14,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 
 *Affecting all Beats*
 
+- Support dashboard loading without Elasticseach {pull}5653[5653]
+
 *Auditbeat*
 
 - Changed `audit.file.path` to be a multi-field so that path is searchable. {pull}5625[5625]

auditbeat/auditbeat.reference.yml

@@ -722,6 +722,9 @@ output.elasticsearch:
 # dashboards and index pattern. Example: testbeat-*
 #setup.dashboards.index:
 
+# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version
+#always_kibana: false
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

filebeat/filebeat.reference.yml

@@ -1150,6 +1150,9 @@ output.elasticsearch:
 # dashboards and index pattern. Example: testbeat-*
 #setup.dashboards.index:
 
+# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version
+#always_kibana: false
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

heartbeat/heartbeat.reference.yml

@@ -842,6 +842,9 @@ output.elasticsearch:
 # dashboards and index pattern. Example: testbeat-*
 #setup.dashboards.index:
 
+# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version
+#always_kibana: false
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

libbeat/_meta/config.reference.yml

@@ -628,6 +628,9 @@ output.elasticsearch:
 # dashboards and index pattern. Example: testbeat-*
 #setup.dashboards.index:
 
+# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version
+#always_kibana: false
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

libbeat/dashboards/config.go

@@ -10,10 +10,12 @@ type Config struct {
 	URL            string `config:"url"`
 	OnlyDashboards bool   `config:"only_dashboards"`
 	OnlyIndex      bool   `config:"only_index"`
+	AlwaysKibana   bool   `config:"always_kibana"`
 }
 
 var defaultConfig = Config{
-	KibanaIndex: ".kibana",
+	KibanaIndex:  ".kibana",
+	AlwaysKibana: false,
 }
 var (
 	defaultDirectory = "kibana"

libbeat/dashboards/dashboards.go

@@ -29,6 +29,14 @@ func ImportDashboards(beatName, hostname, homePath string,
 		return err
 	}
 
+	if kibanaConfig == nil {
+		kibanaConfig = common.NewConfig()
+	}
+
+	if esConfig == nil && dashConfig.AlwaysKibana {
+		return setupAndImportDashboardsViaKibana(hostname, kibanaConfig, &dashConfig, msgOutputter)
+	}
+
 	esLoader, err := NewElasticsearchLoader(esConfig, &dashConfig, msgOutputter)
 	if err != nil {
 		return fmt.Errorf("fail to create the Elasticsearch loader: %v", err)
@@ -48,10 +56,6 @@ func ImportDashboards(beatName, hostname, homePath string,
 
 	logp.Info("For Elasticsearch version >= 6.0.0, the Kibana dashboards need to be imported via the Kibana API.")
 
-	if kibanaConfig == nil {
-		kibanaConfig = common.NewConfig()
-	}
-
 	// In Cloud, the Kibana URL is different than the Elasticsearch URL,
 	// but the credentials are the same.
 	// So, by default, use same credentials for connecting to Kibana as to Elasticsearch
@@ -62,7 +66,13 @@ func ImportDashboards(beatName, hostname, homePath string,
 		kibanaConfig.SetString("password", -1, esLoader.client.Password)
 	}
 
-	kibanaLoader, err := NewKibanaLoader(kibanaConfig, &dashConfig, hostname, msgOutputter)
+	return setupAndImportDashboardsViaKibana(hostname, kibanaConfig, &dashConfig, msgOutputter)
+}
+
+func setupAndImportDashboardsViaKibana(hostname string, kibanaConfig *common.Config,
+	dashboardsConfig *Config, msgOutputter MessageOutputter) error {
+
+	kibanaLoader, err := NewKibanaLoader(kibanaConfig, dashboardsConfig, hostname, msgOutputter)
 	if err != nil {
 		return fmt.Errorf("fail to create the Kibana loader: %v", err)
 	}

libbeat/docs/dashboardsconfig.asciidoc

@@ -91,3 +91,9 @@ is `".kibana"`
 
 The Elasticsearch index name. This setting overwrites the index name defined
 in the dashboards and index pattern. Example: `"testbeat-*"`
+
+[float]
+==== `setup.dashboards.always_kibana`
+
+Force loading of dashboards using the Kibana API without querying Elasticsearch for the version
+The default is `false`.

metricbeat/metricbeat.reference.yml

@@ -1109,6 +1109,9 @@ output.elasticsearch:
 # dashboards and index pattern. Example: testbeat-*
 #setup.dashboards.index:
 
+# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version
+#always_kibana: false
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

packetbeat/packetbeat.reference.yml

@@ -1096,6 +1096,9 @@ output.elasticsearch:
 # dashboards and index pattern. Example: testbeat-*
 #setup.dashboards.index:
 
+# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version
+#always_kibana: false
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

winlogbeat/winlogbeat.reference.yml

@@ -657,6 +657,9 @@ output.elasticsearch:
 # dashboards and index pattern. Example: testbeat-*
 #setup.dashboards.index:
 
+# Force loading of dashboards using the Kibana API without querying Elasticsearch for the version
+#always_kibana: false
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch