Backport: Generate fields.go for netflow (elastic#9740)

* Backport fields generation code

Cherry-pick select parts of f54df4f

* Generate fields.go for netflow input

* Generate fields.go for netflow input

Update x-pack/filebeat to generate a fields.go for each input type. And update the code that
generates the include/list.go to import each input.

* Fix netflow exporter version datatype

It was "int" which is not a valid type. Changed it to long rather than integer because we use that everywhere for numbers.

Cherry-pick parts of c44eb1b (elastic#9686)

* Update x-pack/filebeat to use the xpack.mk

The existing Filebeat Makefile could not be reused because generate_imports.py is not portable.

* Add clean and exportDashboard targets

* createDir in GenerateModuleReferenceConfig

* Remove makefile hack to test x-pack/filebeat

.travis.yml

@@ -35,7 +35,7 @@ jobs:
       go: $GO_VERSION
       stage: test
     - os: linux
-      env: TARGETS="-C x-pack/filebeat unit"
+      env: TARGETS="-C x-pack/filebeat testsuite"
       go: $GO_VERSION
       stage: test
 

Makefile

@@ -17,13 +17,18 @@ XPACK_SUFFIX=x-pack/
 # in the x-pack directory (rather than having the OSS build produce both sets
 # of artifacts). This will be removed once we complete the transition.
 PROJECTS_XPACK_PKG=x-pack/auditbeat
+# PROJECTS_XPACK_MAGE is a list of Beats whose primary build logic is based in
+# Mage. For compatibility with CI testing these projects support a subset of the
+# makefile targets. After all Beats converge to primarily using Mage we can
+# remove this and treat all sub-projects the same.
+PROJECTS_XPACK_MAGE=x-pack/filebeat $(PROJECTS_XPACK_PKG)
 
 # Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection.
 # Also it builds the docs and the generators
 
 .PHONY: testsuite
 testsuite:
-	@$(foreach var,$(PROJECTS),$(MAKE) -C $(var) testsuite || exit 1;)
+	@$(foreach var,$(PROJECTS) $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) testsuite || exit 1;)
 
 .PHONY: setup-commit-hook
 setup-commit-hook:
@@ -59,13 +64,13 @@ coverage-report:
 
 .PHONY: update
 update: notice
-	@$(foreach var,$(PROJECTS),$(MAKE) -C $(var) update || exit 1;)
+	@$(foreach var,$(PROJECTS) $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) update || exit 1;)
 	@$(MAKE) -C deploy/kubernetes all
 
 .PHONY: clean
 clean:
 	@rm -rf build
-	@$(foreach var,$(PROJECTS),$(MAKE) -C $(var) clean || exit 1;)
+	@$(foreach var,$(PROJECTS) $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) clean || exit 1;)
 	@$(MAKE) -C generator clean
 	@-mage -clean 2> /dev/null
 
@@ -77,7 +82,7 @@ clean-vendor:
 
 .PHONY: check
 check: python-env
-	@$(foreach var,$(PROJECTS) dev-tools,$(MAKE) -C $(var) check || exit 1;)
+	@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;)
 	@# Checks also python files which are not part of the beats
 	@$(FIND) -name *.py -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
 	@# Validate that all updates were committed
@@ -112,7 +117,7 @@ misspell:
 
 .PHONY: fmt
 fmt: add-headers python-env
-	@$(foreach var,$(PROJECTS) dev-tools,$(MAKE) -C $(var) fmt || exit 1;)
+	@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) fmt || exit 1;)
 	@# Cleans also python files which are not part of the beats
 	@$(FIND) -name "*.py" -exec $(PYTHON_ENV)/bin/autopep8 --in-place --max-line-length 120 {} \;
 

dev-tools/cmd/module_include_list/module_include_list.go

@@ -18,38 +18,51 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"text/template"
 
+	"github.com/pkg/errors"
+
 	"github.com/elastic/beats/dev-tools/mage"
 	"github.com/elastic/beats/licenses"
 )
 
 var usageText = `
-Usage: module_include_list [flags] [module-dir]
+Usage: module_include_list [flags]
   module_include_list generates a list.go file containing import statements for
-  the module and its dataset packages. The file is usually written to the 
-  include/list.go in the Beat's root directory.
+  the specified imports and module directories. An import is a directory or
+  directory glob containing .go files. A moduleDir is a directory to search
+  for modules and datasets.
+
+  Packages without .go files or without an init() method are omitted from the
+  generated file. The output file is written to the include/list.go in the
+  Beat's root directory by default.
 Options:
 `[1:]
 
 var (
-	license string
-	pkg     string
-	outFile string
+	license    string
+	pkg        string
+	outFile    string
+	moduleDirs stringSliceFlag
+	importDirs stringSliceFlag
 )
 
 func init() {
 	flag.StringVar(&license, "license", "ASL2", "License header for generated file (ASL2 or Elastic).")
 	flag.StringVar(&pkg, "pkg", "include", "Package name.")
 	flag.StringVar(&outFile, "out", "include/list.go", "Output file.")
+	flag.Var(&moduleDirs, "moduleDir", "Directory to search for modules to include")
+	flag.Var(&importDirs, "import", "Directory to include")
 	flag.Usage = usageFlag
 }
 
@@ -62,19 +75,19 @@ func main() {
 		log.Fatalf("Invalid license specifier: %v", err)
 	}
 
-	args := flag.Args()
-	if len(args) != 1 {
-		log.Fatal("module-dir must be passed as an argument.")
+	if len(moduleDirs) == 0 && len(importDirs) == 0 {
+		log.Fatal("At least one -import or -moduleDir must be specified.")
 	}
-	dir := args[0]
 
-	// Find modules and datasets.
-	metaDirs, err := mage.FindFiles(
-		filepath.Join(dir, "*/_meta"),
-		filepath.Join(dir, "*/*/_meta"),
-	)
+	dirs, err := findModuleAndDatasets()
 	if err != nil {
-		log.Fatalf("Failed finding modules and datasets: %v", err)
+		log.Fatal(err)
+	}
+
+	if imports, err := findImports(); err != nil {
+		log.Fatal(err)
+	} else {
+		dirs = append(dirs, imports...)
 	}
 
 	// Get the current directories Go import path.
@@ -85,27 +98,44 @@ func main() {
 
 	// Build import paths.
 	var imports []string
-	for _, metaDir := range metaDirs {
-		importDir := filepath.Dir(metaDir)
-
+	for _, dir := range dirs {
 		// Skip dirs that have no .go files.
-		goFiles, err := filepath.Glob(filepath.Join(importDir, "*.go"))
+		goFiles, err := filepath.Glob(filepath.Join(dir, "*.go"))
 		if err != nil {
 			log.Fatal("Failed checking for .go files in package dir: %v", err)
 		}
 		if len(goFiles) == 0 {
 			continue
 		}
 
-		importDir, err = filepath.Rel(mage.CWD(), filepath.Dir(metaDir))
-		if err != nil {
-			log.Fatal(err)
+		// Skip packages without an init() function because that cannot register
+		// anything as a side-effect of being imported (e.g. filebeat/input/file).
+		var foundInitMethod bool
+		for _, f := range goFiles {
+			if hasInitMethod(f) {
+				foundInitMethod = true
+				break
+			}
+		}
+		if !foundInitMethod {
+			continue
+		}
+
+		importDir := dir
+		if filepath.IsAbs(dir) {
+			// Make it relative to the current package if it's absolute.
+			importDir, err = filepath.Rel(mage.CWD(), dir)
+			if err != nil {
+				log.Fatalf("Failure creating import for dir=%v: %v", dir, err)
+			}
 		}
 
 		imports = append(imports, filepath.ToSlash(
 			filepath.Join(repo.ImportPath, importDir)))
 	}
 
+	sort.Strings(imports)
+
 	// Populate the template.
 	var buf bytes.Buffer
 	err = Template.Execute(&buf, Data{
@@ -155,3 +185,61 @@ type Data struct {
 	Package string
 	Imports []string
 }
+
+//stringSliceFlag is a flag type that allows more than one value to be specified.
+type stringSliceFlag []string
+
+func (f *stringSliceFlag) String() string { return strings.Join(*f, ", ") }
+
+func (f *stringSliceFlag) Set(value string) error {
+	*f = append(*f, value)
+	return nil
+}
+
+// findModuleAndDatasets searches the specified moduleDirs for packages that
+// should be imported. They are designated by the presence of a _meta dir.
+func findModuleAndDatasets() ([]string, error) {
+	var dirs []string
+	for _, moduleDir := range moduleDirs {
+		// Find modules and datasets as indicated by the _meta dir.
+		metaDirs, err := mage.FindFiles(
+			filepath.Join(moduleDir, "*/_meta"),
+			filepath.Join(moduleDir, "*/*/_meta"),
+		)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed finding modules and datasets")
+		}
+
+		for _, metaDir := range metaDirs {
+			// Strip off _meta.
+			dirs = append(dirs, filepath.Dir(metaDir))
+		}
+	}
+	return dirs, nil
+}
+
+// findImports expands the given import values in case they contain globs.
+func findImports() ([]string, error) {
+	return mage.FindFiles(importDirs...)
+}
+
+// hasInitMethod returns true if the file contains 'func init()'.
+func hasInitMethod(file string) bool {
+	f, err := os.Open(file)
+	if err != nil {
+		log.Fatalf("failed to read from %v: %v", file, err)
+	}
+	defer f.Close()
+
+	var initSignature = []byte("func init()")
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		if bytes.Contains(scanner.Bytes(), initSignature) {
+			return true
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		log.Fatal("failed scanning %v: %v", file, err)
+	}
+	return false
+}

dev-tools/mage/common.go

@@ -676,6 +676,23 @@ func IsUpToDate(dst string, sources ...string) bool {
 	return err == nil && !execute
 }
 
+// OSSBeatDir returns the OSS beat directory. You can pass paths and they will
+// be joined and appended to the OSS beat dir.
+func OSSBeatDir(path ...string) string {
+	ossDir := CWD()
+
+	// Check if we need to correct ossDir because it's in x-pack.
+	if parentDir := filepath.Base(filepath.Dir(ossDir)); parentDir == "x-pack" {
+		// If the OSS version of the beat exists.
+		tmp := filepath.Join(ossDir, "../..", BeatName)
+		if _, err := os.Stat(tmp); !os.IsNotExist(err) {
+			ossDir = tmp
+		}
+	}
+
+	return filepath.Join(append([]string{ossDir}, path...)...)
+}
+
 // LibbeatDir returns the libbeat directory. You can pass paths and
 // they will be joined and appended to the libbeat dir.
 func LibbeatDir(path ...string) string {

dev-tools/mage/config.go

@@ -151,5 +151,5 @@ func GenerateModuleReferenceConfig(out string, moduleDirs ...string) error {
 		"Modules": moduleConfigs,
 	})
 
-	return ioutil.WriteFile(out, []byte(config), 0644)
+	return ioutil.WriteFile(createDir(out), []byte(config), 0644)
 }