Include Kibana dashboards in the individual Beats packages (#4675)

* Include Kibana dashboards in the individual Beats packages

* include the kibana directory in the home path of the packages
* make the `setup.dashboards.directory` the default instead of the URL
* remove the `snapshot.url` and `snapshot` options
* keep the `setup.dashboards.url`, but without a default value
* keep the beats-dashboards package for the moment, but it won't be easy to use (you
  need to know the URL).

.gitignore

@@ -5,7 +5,6 @@
 /build
 /*/data
 /*/logs
-/*/_meta/kibana/index-pattern
 /*/fields.yml
 /*/*.template*.json
 

CHANGELOG.asciidoc

@@ -20,6 +20,8 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha2...master[Check the HEAD d
 - Change format of the saved kibana dashboards to have a single JSON file for each dashboard {pull}4413[4413]
 - Rename `configtest` command to `test config`. {pull}4590[4590]
 - Remove setting `queue_size` and `bulk_queue_size`. {pull}4650[4650]
+- Remove setting `dashboard.snapshot` and `dashboard.snapshot_url`. They are no longer needed because the
+  dashboards are included in the packages by default. {pull}4675[4675]
 
 *Filebeat*
 
@@ -79,6 +81,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha2...master[Check the HEAD d
 - Introduce configurable event queue settings: queue.mem.events, queue.mem.flush.min_events and queue.mem.flush.timeout. {pull}4650[4650]
 - Enable pipelining in Logstash output by default. {pull}4650[4650]
 - Added 'result' field to Elasticsearch QueryResult struct for compatibility with 6.x Index and Delete API responses. {issue]4661[4661]
+- The sample dashboards are now included in the Beats packages. {pull}4675[4675]
 
 *Filebeat*
 

auditbeat/auditbeat.reference.yml

@@ -640,27 +640,18 @@ output.elasticsearch:
 # options here, or by using the `-setup` CLI flag or the `setup` command.
 #setup.dashboards.enabled: false
 
-# The URL from where to download the dashboards archive. By default this URL
-# has a value which is computed based on the Beat name and version. For released
-# versions, this URL points to the dashboard archive on the artifacts.elastic.co
-# website.
-#setup.dashboards.url:
+# The directory from where to read the dashboards. The default is the `kibana`
+# folder in the home path.
+#setup.dashboards.directory: ${path.home}/kibana
 
-# The directory from where to read the dashboards. It is used instead of the URL
-# when it has a value.
-#setup.dashboards.directory:
+# The URL from where to download the dashboards archive. It is used instead of
+# the directory if it has a value.
+#setup.dashboards.url:
 
 # The file archive (zip file) from where to read the dashboards. It is used instead
-# of the URL when it has a value.
+# of the directory when it has a value.
 #setup.dashboards.file:
 
-# If this option is enabled, the snapshot URL is used instead of the default URL.
-#setup.dashboards.snapshot: false
-
-# The URL from where to download the snapshot version of the dashboards. By default
-# this has a value which is computed based on the Beat name and version.
-#setup.dashboards.snapshot_url
-
 # In case the archive contains the dashboards from multiple Beats, this lets you
 # select which one to load. You can load all the dashboards in the archive by
 # setting this to the empty string.

filebeat/filebeat.reference.yml

@@ -1012,27 +1012,18 @@ output.elasticsearch:
 # options here, or by using the `-setup` CLI flag or the `setup` command.
 #setup.dashboards.enabled: false
 
-# The URL from where to download the dashboards archive. By default this URL
-# has a value which is computed based on the Beat name and version. For released
-# versions, this URL points to the dashboard archive on the artifacts.elastic.co
-# website.
-#setup.dashboards.url:
+# The directory from where to read the dashboards. The default is the `kibana`
+# folder in the home path.
+#setup.dashboards.directory: ${path.home}/kibana
 
-# The directory from where to read the dashboards. It is used instead of the URL
-# when it has a value.
-#setup.dashboards.directory:
+# The URL from where to download the dashboards archive. It is used instead of
+# the directory if it has a value.
+#setup.dashboards.url:
 
 # The file archive (zip file) from where to read the dashboards. It is used instead
-# of the URL when it has a value.
+# of the directory when it has a value.
 #setup.dashboards.file:
 
-# If this option is enabled, the snapshot URL is used instead of the default URL.
-#setup.dashboards.snapshot: false
-
-# The URL from where to download the snapshot version of the dashboards. By default
-# this has a value which is computed based on the Beat name and version.
-#setup.dashboards.snapshot_url
-
 # In case the archive contains the dashboards from multiple Beats, this lets you
 # select which one to load. You can load all the dashboards in the archive by
 # setting this to the empty string.

heartbeat/heartbeat.reference.yml

@@ -790,27 +790,18 @@ output.elasticsearch:
 # options here, or by using the `-setup` CLI flag or the `setup` command.
 #setup.dashboards.enabled: false
 
-# The URL from where to download the dashboards archive. By default this URL
-# has a value which is computed based on the Beat name and version. For released
-# versions, this URL points to the dashboard archive on the artifacts.elastic.co
-# website.
-#setup.dashboards.url:
+# The directory from where to read the dashboards. The default is the `kibana`
+# folder in the home path.
+#setup.dashboards.directory: ${path.home}/kibana
 
-# The directory from where to read the dashboards. It is used instead of the URL
-# when it has a value.
-#setup.dashboards.directory:
+# The URL from where to download the dashboards archive. It is used instead of
+# the directory if it has a value.
+#setup.dashboards.url:
 
 # The file archive (zip file) from where to read the dashboards. It is used instead
-# of the URL when it has a value.
+# of the directory when it has a value.
 #setup.dashboards.file:
 
-# If this option is enabled, the snapshot URL is used instead of the default URL.
-#setup.dashboards.snapshot: false
-
-# The URL from where to download the snapshot version of the dashboards. By default
-# this has a value which is computed based on the Beat name and version.
-#setup.dashboards.snapshot_url
-
 # In case the archive contains the dashboards from multiple Beats, this lets you
 # select which one to load. You can load all the dashboards in the archive by
 # setting this to the empty string.

libbeat/_meta/config.reference.yml

@@ -576,27 +576,18 @@ output.elasticsearch:
 # options here, or by using the `-setup` CLI flag or the `setup` command.
 #setup.dashboards.enabled: false
 
-# The URL from where to download the dashboards archive. By default this URL
-# has a value which is computed based on the Beat name and version. For released
-# versions, this URL points to the dashboard archive on the artifacts.elastic.co
-# website.
-#setup.dashboards.url:
+# The directory from where to read the dashboards. The default is the `kibana`
+# folder in the home path.
+#setup.dashboards.directory: ${path.home}/kibana
 
-# The directory from where to read the dashboards. It is used instead of the URL
-# when it has a value.
-#setup.dashboards.directory:
+# The URL from where to download the dashboards archive. It is used instead of
+# the directory if it has a value.
+#setup.dashboards.url:
 
 # The file archive (zip file) from where to read the dashboards. It is used instead
-# of the URL when it has a value.
+# of the directory when it has a value.
 #setup.dashboards.file:
 
-# If this option is enabled, the snapshot URL is used instead of the default URL.
-#setup.dashboards.snapshot: false
-
-# The URL from where to download the snapshot version of the dashboards. By default
-# this has a value which is computed based on the Beat name and version.
-#setup.dashboards.snapshot_url
-
 # In case the archive contains the dashboards from multiple Beats, this lets you
 # select which one to load. You can load all the dashboards in the archive by
 # setting this to the empty string.

libbeat/beat/beat.go

@@ -575,8 +575,8 @@ func (b *Beat) loadDashboards(force bool) error {
 		if b.Config.Output.Name() == "elasticsearch" {
 			esConfig = b.Config.Output.Config()
 		}
-		err := dashboards.ImportDashboards(b.Info.Beat, b.Info.Version, b.Config.Kibana,
-			esConfig, b.Config.Dashboards)
+		err := dashboards.ImportDashboards(b.Info.Beat, b.Info.Version, paths.Resolve(paths.Home, ""),
+			b.Config.Kibana, esConfig, b.Config.Dashboards)
 		if err != nil {
 			return fmt.Errorf("Error importing Kibana dashboards: %v", err)
 		}

libbeat/dashboards/config.go

@@ -10,14 +10,11 @@ type Config struct {
 	URL            string `config:"url"`
 	OnlyDashboards bool   `config:"only_dashboards"`
 	OnlyIndex      bool   `config:"only_index"`
-	Snapshot       bool   `config:"snapshot"`
-	SnapshotURL    string `config:"snapshot_url"`
 }
 
 var defaultConfig = Config{
 	KibanaIndex: ".kibana",
 }
 var (
-	defaultURLPattern  = "https://artifacts.elastic.co/downloads/beats/beats-dashboards/beats-dashboards-%s.zip"
-	snapshotURLPattern = "https://snapshots.elastic.co/downloads/beats/beats-dashboards/beats-dashboards-%s-SNAPSHOT.zip"
+	defaultDirectory = "kibana"
 )

libbeat/dashboards/dashboards.go

@@ -2,23 +2,25 @@ package dashboards
 
 import (
 	"fmt"
+	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
 )
 
-func ImportDashboards(beatName, beatVersion string, kibanaConfig *common.Config, esConfig *common.Config,
+func ImportDashboards(beatName, beatVersion, homePath string, kibanaConfig *common.Config, esConfig *common.Config,
 	dashboardsConfig *common.Config) error {
 	if dashboardsConfig == nil || !dashboardsConfig.Enabled() {
 		return nil
 	}
 
 	dashConfig := defaultConfig
 	dashConfig.Beat = beatName
-	dashConfig.URL = fmt.Sprintf(defaultURLPattern, beatVersion)
-	dashConfig.SnapshotURL = fmt.Sprintf(snapshotURLPattern, beatVersion)
+	if dashConfig.Dir == "" {
+		dashConfig.Dir = filepath.Join(homePath, defaultDirectory)
+	}
 
 	err := dashboardsConfig.Unpack(&dashConfig)
 	if err != nil {

libbeat/dashboards/importer.go

@@ -41,19 +41,16 @@ func NewImporter(version string, cfg *Config, loader Loader) (*Importer, error)
 
 // Import imports the Kibana dashboards according to the configuration options.
 func (imp Importer) Import() error {
-	if imp.cfg.Dir != "" {
-		err := imp.ImportKibanaDir(imp.cfg.Dir)
+
+	if imp.cfg.URL != "" || imp.cfg.File != "" {
+		err := imp.ImportArchive()
 		if err != nil {
-			return fmt.Errorf("Error importing directory %s: %v", imp.cfg.Dir, err)
+			return fmt.Errorf("Error importing URL/file: %v", err)
 		}
 	} else {
-		if imp.cfg.URL != "" || imp.cfg.Snapshot || imp.cfg.File != "" {
-			err := imp.ImportArchive()
-			if err != nil {
-				return fmt.Errorf("Error importing URL/file: %v", err)
-			}
-		} else {
-			return fmt.Errorf("No URL and no file specify. Nothing to import")
+		err := imp.ImportKibanaDir(imp.cfg.Dir)
+		if err != nil {
+			return fmt.Errorf("Error importing directory %s: %v", imp.cfg.Dir, err)
 		}
 	}
 	return nil
@@ -161,13 +158,6 @@ func (imp Importer) ImportArchive() error {
 	imp.loader.statusMsg("Created temporary directory %s", target)
 	if imp.cfg.File != "" {
 		archive = imp.cfg.File
-	} else if imp.cfg.Snapshot {
-		// In case snapshot is set, snapshot version is fetched
-		url := imp.cfg.SnapshotURL
-		archive, err = imp.downloadFile(url, target)
-		if err != nil {
-			return fmt.Errorf("Failed to download snapshot file: %s. Error: %v", url, err)
-		}
 	} else if imp.cfg.URL != "" {
 		archive, err = imp.downloadFile(imp.cfg.URL, target)
 		if err != nil {

libbeat/docs/dashboardsconfig.asciidoc

@@ -43,49 +43,32 @@ You can specify the following options in the `dashboards` section of the
 ===== enabled
 
 If this option is set to true, {beatname_uc} loads the sample Kibana dashboards
-automatically on startup. If no other options are set, the dashboard archive is
-downloaded from the elastic.co website.
+automatically on startup. If no other options are set, the dashboard are loaded
+from the local `kibana` directory in the home path of the installation.
 
-To load dashboards from a different location, you can
-configure one of the following options: <<url-option,`url`>>,
-<<directory-option,`directory`>>, or <<file-option,`file`>>.
+To load dashboards from a different location, you can configure one of the
+following options: <<url-option,`url`>>, or <<file-option,`file`>>.
 
-To load dashboards from a snapshot URL, use the <<snapshot-option,`snapshot`>>
-option and optionally <<snapshot-url-option,`snapshot-url`>>.
+[[directory-option]]
+===== directory
+
+The directory that contains the dashboards to load. The default is the `kibana`
+folder in the home path.
 
 [[url-option]]
 ===== url
 
-The URL to use for downloading the dashboard archive. By default this URL
-is computed based on the Beat name and version. For released versions,
-this URL points to the dashboard archive on the artifacts.elastic.co
-website.
-
-[[directory-option]]
-===== directory
+The URL to use for downloading the dashboard archive. If this option
+is set, {beatname_uc} downloads the dashboard archive from the specified URL
+instead of using the local directory.
 
-The directory that contains the dashboards to load. If this option is set,
-{beatname_uc} looks for dashboards in the specified directory instead of
-downloading an archive from a URL. 
 
 [[file-option]]
 ===== file
 
 The file archive (zip file) that contains the dashboards to load. If this option
 is set, {beatname_uc} looks for a dashboard archive in the specified path
-instead of downloading the archive from a URL. 
-
-[[snapshot-option]]
-===== snapshot
-
-If this option is set to true, the snapshot URL is used instead of the default
-URL.
-
-[[snapshot-url-option]]
-===== snapshot_url
-
-The URL to use for downloading the snapshot version of the dashboards. By
-default the snapshot URL is computed based on the Beat name and version.
+instead of using the local directory.
 
 ===== beat
 

libbeat/scripts/Makefile

@@ -356,6 +356,10 @@ install-home:
 		rsync -av _meta/module.generated/ ${HOME_PREFIX}/module/; \
 		chmod -R go-w ${HOME_PREFIX}/module/; \
 	fi
+	if [ -d _meta/kibana ]; then \
+		install -d -m 755 ${HOME_PREFIX}/kibana; \
+		rsync -av _meta/kibana/ ${HOME_PREFIX}/kibana/; \
+	fi
 
 # Prepares for packaging. Builds binaries and creates homedir data
 .PHONY: prepare-package

metricbeat/metricbeat.reference.yml

@@ -985,27 +985,18 @@ output.elasticsearch:
 # options here, or by using the `-setup` CLI flag or the `setup` command.
 #setup.dashboards.enabled: false
 
-# The URL from where to download the dashboards archive. By default this URL
-# has a value which is computed based on the Beat name and version. For released
-# versions, this URL points to the dashboard archive on the artifacts.elastic.co
-# website.
-#setup.dashboards.url:
+# The directory from where to read the dashboards. The default is the `kibana`
+# folder in the home path.
+#setup.dashboards.directory: ${path.home}/kibana
 
-# The directory from where to read the dashboards. It is used instead of the URL
-# when it has a value.
-#setup.dashboards.directory:
+# The URL from where to download the dashboards archive. It is used instead of
+# the directory if it has a value.
+#setup.dashboards.url:
 
 # The file archive (zip file) from where to read the dashboards. It is used instead
-# of the URL when it has a value.
+# of the directory when it has a value.
 #setup.dashboards.file:
 
-# If this option is enabled, the snapshot URL is used instead of the default URL.
-#setup.dashboards.snapshot: false
-
-# The URL from where to download the snapshot version of the dashboards. By default
-# this has a value which is computed based on the Beat name and version.
-#setup.dashboards.snapshot_url
-
 # In case the archive contains the dashboards from multiple Beats, this lets you
 # select which one to load. You can load all the dashboards in the archive by
 # setting this to the empty string.