Update Filebeat's magefile.go (#9700)

* Update Filebeat's magefile.go

This improves and fixes a few issues.

- Generate field docs that include fields from x-pack/filebeat.
- Generate fields.go for each Filebeat module.
- Add direct packaging build targets to x-pack/filebeat (packaging is no longer done via OSS filebeat).

* Add goTestUnit alias

Makefile

@@ -16,12 +16,12 @@ XPACK_SUFFIX=x-pack/
 # PROJECTS_XPACK_PKG is a list of Beats that have independent packaging support
 # in the x-pack directory (rather than having the OSS build produce both sets
 # of artifacts). This will be removed once we complete the transition.
-PROJECTS_XPACK_PKG=x-pack/auditbeat
+PROJECTS_XPACK_PKG=x-pack/auditbeat x-pack/filebeat
 # PROJECTS_XPACK_MAGE is a list of Beats whose primary build logic is based in
 # Mage. For compatibility with CI testing these projects support a subset of the
 # makefile targets. After all Beats converge to primarily using Mage we can
 # remove this and treat all sub-projects the same.
-PROJECTS_XPACK_MAGE=x-pack/filebeat x-pack/metricbeat $(PROJECTS_XPACK_PKG)
+PROJECTS_XPACK_MAGE=x-pack/metricbeat $(PROJECTS_XPACK_PKG)
 
 # Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection.
 # Also it builds the docs and the generators

auditbeat/magefile.go

@@ -131,13 +131,13 @@ func Docs() {
 
 // combinedDocs builds combined documentation for both OSS and X-Pack.
 func combinedDocs() error {
-	return auditbeat.CollectDocs(mage.OSSBeatDir(), auditbeat.XpackBeatDir())
+	return auditbeat.CollectDocs(mage.OSSBeatDir(), mage.XPackBeatDir())
 }
 
 // xpackFields creates x-pack/auditbeat/fields.yml - necessary to build
 // a combined documentation.
 func xpackFields() error {
-	return mage.Mage(auditbeat.XpackBeatDir(), "fields")
+	return mage.Mage(mage.XPackBeatDir(), "fields")
 }
 
 // Fmt formats source code and adds file headers.

auditbeat/scripts/mage/docs.go

@@ -86,16 +86,5 @@ func CollectDocs(basePaths ...string) error {
 		return err
 	}
 
-	esBeats, err := mage.ElasticBeatsDir()
-	if err != nil {
-		return err
-	}
-
-	return sh.Run(python, mage.LibbeatDir("scripts/generate_fields_docs.py"),
-		XpackBeatDir(), mage.BeatName, esBeats, "--output_path", mage.OSSBeatDir())
-}
-
-// XpackBeatDir returns the x-pack/{beatname} directory for a Beat.
-func XpackBeatDir() string {
-	return mage.OSSBeatDir("../x-pack", mage.BeatName)
+	return mage.Docs.FieldDocs(mage.XPackBeatDir("fields.yml"))
 }

dev-tools/mage/common.go

@@ -693,6 +693,12 @@ func OSSBeatDir(path ...string) string {
 	return filepath.Join(append([]string{ossDir}, path...)...)
 }
 
+// XPackBeatDir returns the X-Pack beat directory. You can pass paths and they
+// will be joined and appended to the X-Pack beat dir.
+func XPackBeatDir(path ...string) string {
+	return OSSBeatDir(append([]string{XPackDir, BeatName}, path...)...)
+}
+
 // LibbeatDir returns the libbeat directory. You can pass paths and
 // they will be joined and appended to the libbeat dir.
 func LibbeatDir(path ...string) string {
@@ -705,7 +711,13 @@ func LibbeatDir(path ...string) string {
 }
 
 // createDir creates the parent directory for the given file.
+// Deprecated: Use CreateDir.
 func createDir(file string) string {
+	return CreateDir(file)
+}
+
+// CreateDir creates the parent directory for the given file.
+func CreateDir(file string) string {
 	// Create the output directory.
 	if dir := filepath.Dir(file); dir != "." {
 		if err := os.MkdirAll(dir, 0755); err != nil {

dev-tools/mage/config.go

@@ -27,6 +27,8 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/magefile/mage/mg"
+
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 )
@@ -63,8 +65,11 @@ func (t ConfigFileType) IsDocker() bool { return t&DockerConfigType > 0 }
 // ConfigFileParams defines the files that make up each config file.
 type ConfigFileParams struct {
 	ShortParts     []string // List of files or globs.
+	ShortDeps      []interface{}
 	ReferenceParts []string // List of files or globs.
+	ReferenceDeps  []interface{}
 	DockerParts    []string // List of files or globs.
+	DockerDeps     []interface{}
 	ExtraVars      map[string]interface{}
 }
 
@@ -122,18 +127,21 @@ func makeConfigTemplates(types ConfigFileType, args ConfigFileParams) error {
 	var err error
 
 	if types.IsShort() {
+		mg.SerialDeps(args.ShortDeps...)
 		if err = makeConfigTemplate(shortTemplate, 0600, args.ShortParts...); err != nil {
 			return err
 		}
 	}
 
 	if types.IsReference() {
+		mg.SerialDeps(args.ReferenceDeps...)
 		if err = makeConfigTemplate(referenceTemplate, 0644, args.ReferenceParts...); err != nil {
 			return err
 		}
 	}
 
 	if types.IsDocker() {
+		mg.SerialDeps(args.DockerDeps...)
 		if err = makeConfigTemplate(dockerTemplate, 0600, args.DockerParts...); err != nil {
 			return err
 		}

dev-tools/mage/docs.go

@@ -19,7 +19,6 @@ package mage
 
 import (
 	"log"
-	"path/filepath"
 
 	"github.com/magefile/mage/sh"
 )
@@ -49,7 +48,7 @@ func (b docsBuilder) FieldDocs(fieldsYML string) error {
 
 	log.Println(">> Generating docs/fields.asciidoc for", BeatName)
 	return sh.Run(python, LibbeatDir("scripts/generate_fields_docs.py"),
-		filepath.Dir(fieldsYML),       // Path to dir containing fields.yml.
+		fieldsYML,                     // Path to fields.yml.
 		BeatName,                      // Beat title.
 		esBeats,                       // Path to general beats folder.
 		"--output_path", OSSBeatDir()) // It writes to {output_path}/docs/fields.asciidoc.

dev-tools/mage/fields.go

@@ -32,10 +32,17 @@ import (
 // moduleDirs specifies additional directories to search for modules. The
 // contents of each fields.yml will be included in the generated file.
 func GenerateFieldsYAML(moduleDirs ...string) error {
-	return generateFieldsYAML(OSSBeatDir(), moduleDirs...)
+	return generateFieldsYAML(OSSBeatDir(), "fields.yml", moduleDirs...)
 }
 
-func generateFieldsYAML(baseDir string, moduleDirs ...string) error {
+// GenerateFieldsYAMLTo generates a YAML file containing the field definitions
+// for the Beat. It's the same as GenerateFieldsYAML but with a configurable
+// output file.
+func GenerateFieldsYAMLTo(output string, moduleDirs ...string) error {
+	return generateFieldsYAML(OSSBeatDir(), output, moduleDirs...)
+}
+
+func generateFieldsYAML(baseDir, output string, moduleDirs ...string) error {
 	const globalFieldsCmdPath = "libbeat/scripts/cmd/global_fields/main.go"
 
 	beatsDir, err := ElasticBeatsDir()
@@ -47,7 +54,7 @@ func generateFieldsYAML(baseDir string, moduleDirs ...string) error {
 		filepath.Join(beatsDir, globalFieldsCmdPath),
 		"-es_beats_path", beatsDir,
 		"-beat_path", baseDir,
-		"-out", "fields.yml",
+		"-out", output,
 	)
 
 	return globalFieldsCmd(moduleDirs...)

dev-tools/mage/fmt.go

@@ -79,7 +79,9 @@ func GoImports() error {
 // ignores build/ directories.
 func PythonAutopep8() error {
 	pyFiles, err := FindFilesRecursive(func(path string, _ os.FileInfo) bool {
-		return filepath.Ext(path) == ".py" && !strings.Contains(path, "build/")
+		return filepath.Ext(path) == ".py" &&
+			!strings.Contains(path, "build/") &&
+			!strings.Contains(path, "vendor/")
 	})
 	if err != nil {
 		return err

filebeat/Makefile

@@ -4,60 +4,25 @@ SYSTEM_TESTS?=true
 TEST_ENVIRONMENT?=true
 GOX_FLAGS=-arch="amd64 386 arm ppc64 ppc64le"
 ES_BEATS?=..
-FIELDS_FILE_PATH=module
-
-DOCS_BRANCH=$(shell grep doc-branch ../libbeat/docs/version.asciidoc | cut -c 14-)
+EXCLUDE_COMMON_UPDATE_TARGET=true
 
 include ${ES_BEATS}/libbeat/scripts/Makefile
 
-# Collects all module dashboards
-.PHONY: kibana
-kibana:
-	@rm -rf _meta/kibana.generated
-	@mkdir -p _meta/kibana.generated
-	@-cp -pr module/*/_meta/kibana/* _meta/kibana.generated
-
-# Collects all module configs
-.PHONY: configs
-configs: python-env
-	@cp ${ES_BEATS}/filebeat/_meta/common.p1.yml _meta/beat.yml
-	@cat ${ES_BEATS}/filebeat/_meta/common.p2.yml >> _meta/beat.yml
-	@cat ${ES_BEATS}/filebeat/_meta/common.reference.p1.yml > _meta/beat.reference.yml
-	@${PYTHON_ENV}/bin/python  ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
-	@cat ${ES_BEATS}/filebeat/_meta/common.reference.inputs.yml >> _meta/beat.reference.yml
-	@cat ${ES_BEATS}/filebeat/_meta/common.reference.p2.yml >> _meta/beat.reference.yml
-	@rm -rf modules.d
-	${PYTHON_ENV}/bin/python ${ES_BEATS}/script/modules_collector.py --beat ${BEAT_NAME} --docs_branch=$(DOCS_BRANCH)
-	@chmod go-w modules.d/*
-
-# Collects all module docs
-.PHONY: collect-docs
-collect-docs: python-env
-	@rm -rf docs/modules
-	@mkdir -p docs/modules
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/filebeat/scripts/docs_collector.py --beat ${BEAT_NAME}
-
-# Generate imports for inputs
-.PHONY: imports
-imports: python-env
-	@mkdir -p include
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/generate_imports.py ${BEAT_PATH}
-
-# Runs all collection steps and updates afterwards
-.PHONY: collect
-collect: kibana configs collect-docs imports
+.PHONY: update
+update: mage
+	mage update
 
-# Creates a new module. Requires the params MODULE
+# Creates a new module. Requires the params MODULE.
 .PHONY: create-module
 create-module:
 	@go run ${ES_BEATS}/filebeat/scripts/generator/module/main.go --path=$(PWD) --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE)
 
-# Creates a new fileset. Requires the params MODULE and FILESET
+# Creates a new fileset. Requires the params MODULE and FILESET.
 .PHONY: create-fileset
 create-fileset:
 	@go run ${ES_BEATS}/filebeat/scripts/generator/fileset/main.go --path=$(PWD) --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE) --fileset=$(FILESET)
 
-# Creates a fields.yml based on a pipeline.json file. Requires the params MODULE and FILESET
+# Creates a fields.yml based on a pipeline.json file. Requires the params MODULE and FILESET.
 .PHONY: create-fields
 create-fields:
 	@go run ${ES_BEATS}/filebeat/scripts/generator/fields/main.go --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE) --fileset=$(FILESET)

filebeat/README.md

@@ -30,7 +30,3 @@ If you are sure you found a bug or have a feature request, open an issue on
 
 We love contributions from our community! Please read the
 [CONTRIBUTING.md](../CONTRIBUTING.md) file.
-
-## Snapshots
-
-For testing purposes, we generate snapshot builds that you can find [here](https://beats-nightlies.s3.amazonaws.com/index.html?prefix=filebeat). Please be aware that these are built on top of master and are not meant for production.

filebeat/docker-compose.yml

@@ -5,9 +5,13 @@ services:
     depends_on:
       - proxy_dep
     env_file:
-      - ${PWD}/build/test.env
       - ${PWD}/input/redis/_meta/env
     environment:
+      - BEAT_STRICT_PERMS=false
+      - ES_HOST=elasticsearch
+      - ES_PORT=9200
+      - ES_USER=beats
+      - ES_PASS=testing
       - KIBANA_HOST=kibana
       - KIBANA_PORT=5601
     working_dir: /go/src/github.com/elastic/beats/filebeat