generate module.yml for create-fileset (#5657) (#6157)

* generate module.yml for create-fileset command and fixes bugs there (#5657)

* create fileset for already created module

* automatically configure module.yml when add fileset

* add docs about module.yml file

CHANGELOG.asciidoc

@@ -79,6 +79,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 
 *Filebeat*
 
+- Right check of module/fileset exists for create-fileset command {issue}5657[5657]
+- Place fields.yml and docs.asciidoc to _meta folder at create-fileset command {issue}5657[5657]
 - Add support for adding string tags {pull}5395{5395}
 - Fix race condition when limiting the number of harvesters running in parallel {issue}5458[5458]
 - Fix relative paths in the prospector definitions. {pull}5443[5433]
@@ -158,6 +160,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 
 *Filebeat*
 
+- Create module.yml at create-fileset command {issue}5657[5657]
 - Add PostgreSQL module with slowlog support. {pull}4763[4763]
 - Add Kafka log module. {pull}4885[4885]
 - Add support for `/var/log/containers/` log path in `add_kubernetes_metadata` processor. {pull}4981[4981]

docs/devguide/modules-dev-guide.asciidoc

@@ -296,6 +296,14 @@ run.
 Besides the files in the fileset folder, there is also data that needs to be
 filled at the module level.
 
+[float]
+==== module.yml
+
+This file contains list of all the dashboards available for the module and used by `export_dashboards.go` script for exporting dashboards.
+Each dashboard is defined by an id and the name of json file where the dashboard is saved locally.
+At generation new fileset this file will be automatically updated with "default" dashboard settings for new fileset.
+Please ensure that this settings are correct.
+
 [float]
 ==== _meta/docs.asciidoc
 

filebeat/scripts/generator/fileset/main.go

@@ -11,26 +11,56 @@ import (
 
 func copyTemplatesToDest(templatesPath, name, filesetPath, module, fileset string) error {
 	template := path.Join(templatesPath, name)
+	dest := path.Join(filesetPath, name)
+	return copyTemplate(template, dest, module, fileset)
+}
+
+func readTemplate(template, module, fileset string) ([]byte, error) {
 	c, err := ioutil.ReadFile(template)
 	if err != nil {
-		return err
+		return []byte{}, fmt.Errorf("cannot read template: %v", err)
 	}
 
 	c = bytes.Replace(c, []byte("{module}"), []byte(module), -1)
 	c = bytes.Replace(c, []byte("{fileset}"), []byte(fileset), -1)
 
-	dest := path.Join(filesetPath, name)
+	return c, nil
+}
+
+func copyTemplate(template, dest, module, fileset string) error {
+	c, err := readTemplate(template, module, fileset)
+	if err != nil {
+		return err
+	}
+
 	err = ioutil.WriteFile(dest, c, os.ModePerm)
 	if err != nil {
 		return fmt.Errorf("cannot copy template: %v", err)
 	}
 	return nil
 }
 
+func appendTemplate(template, dest, module, fileset string) error {
+	c, err := readTemplate(template, module, fileset)
+	if err != nil {
+		return err
+	}
+
+	f, err := os.OpenFile(dest, os.O_WRONLY|os.O_APPEND, os.ModePerm)
+	if err == nil {
+		_, err = f.Write(c)
+	}
+	if err != nil {
+		return fmt.Errorf("cannot append template: %v", err)
+	}
+
+	return nil
+}
+
 func generateModule(module, fileset, modulePath, beatsPath string) error {
 	p := path.Join(modulePath, "module", module)
-	if _, err := os.Stat(p); os.IsExist(err) {
-		return fmt.Errorf("module already exists: %s at %s", module, p)
+	if _, err := os.Stat(p); !os.IsNotExist(err) {
+		return nil
 	}
 
 	d := path.Join(p, "_meta", "kibana", "default")
@@ -40,9 +70,9 @@ func generateModule(module, fileset, modulePath, beatsPath string) error {
 	}
 
 	templatesPath := path.Join(beatsPath, "scripts", "module")
-	filesToCopy := []string{path.Join("fields.yml"), path.Join("docs.asciidoc")}
+	filesToCopy := []string{path.Join("_meta", "fields.yml"), path.Join("_meta", "docs.asciidoc"), path.Join("module.yml")}
 	for _, f := range filesToCopy {
-		err := copyTemplatesToDest(templatesPath, f, p, module, "")
+		err := copyTemplatesToDest(templatesPath, f, p, module, fileset)
 		if err != nil {
 			return err
 		}
@@ -53,7 +83,7 @@ func generateModule(module, fileset, modulePath, beatsPath string) error {
 
 func generateFileset(module, fileset, modulePath, beatsPath string) error {
 	filesetPath := path.Join(modulePath, "module", module, fileset)
-	if _, err := os.Stat(filesetPath); os.IsExist(err) {
+	if _, err := os.Stat(filesetPath); !os.IsNotExist(err) {
 		return fmt.Errorf("fileset already exists: %s", fileset)
 	}
 
@@ -74,7 +104,15 @@ func generateFileset(module, fileset, modulePath, beatsPath string) error {
 			return err
 		}
 	}
-	return nil
+
+	return addFilesetDashboard(module, fileset, modulePath, beatsPath)
+}
+
+func addFilesetDashboard(module, fileset, modulePath, beatsPath string) error {
+	templatesPath := path.Join(beatsPath, "scripts", "module")
+	template := path.Join(templatesPath, "module-fileset.yml")
+	dest := path.Join(modulePath, "module", module, "module.yml")
+	return appendTemplate(template, dest, module, fileset)
 }
 
 func main() {
@@ -106,5 +144,5 @@ func main() {
 		os.Exit(3)
 	}
 
-	fmt.Println("New module was generated. After setting up Grok pattern in pipeline.json, please generate fields.yml")
+	fmt.Println("New module was generated, please check that module.yml file have proper fileset dashboard settings. After setting up Grok pattern in pipeline.json, please generate fields.yml")
 }

filebeat/scripts/module/module-fileset.yml

@@ -0,0 +1,2 @@
+- id: Filebeat-{module}-{fileset}-Dashboard
+  file: Filebeat-{module}-{fileset}.json

filebeat/scripts/module/module.yml

@@ -0,0 +1 @@
+dashboards: