Update Packetbeat's magefile.go (#9662)

Enhance Packetbeat's magefile.go to be responsible for all aspects of the
`update` target (fields.yml, fields.go, config, include/list.go, and field
docs).

This introduces a fields.go file for each protocol in Packetbeat. The
include/fields.go file contains the libbeat fields and the common fields
from Packetbeat.

dev-tools/cmd/module_include_list/module_include_list.go

@@ -18,6 +18,7 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"flag"
 	"fmt"
@@ -28,28 +29,39 @@ import (
 	"strings"
 	"text/template"
 
+	"github.com/pkg/errors"
+
 	"github.com/elastic/beats/dev-tools/mage"
 	"github.com/elastic/beats/licenses"
 )
 
 var usageText = `
-Usage: module_include_list [flags] [module-dir]
+Usage: module_include_list [flags]
   module_include_list generates a list.go file containing import statements for
-  the module and its dataset packages. The file is usually written to the 
-  include/list.go in the Beat's root directory.
+  the specified imports and module directories. An import is a directory or
+  directory glob containing .go files. A moduleDir is a directory to search
+  for modules and datasets.
+
+  Packages without .go files or without an init() method are omitted from the
+  generated file. The output file is written to the include/list.go in the
+  Beat's root directory by default.
 Options:
 `[1:]
 
 var (
-	license string
-	pkg     string
-	outFile string
+	license    string
+	pkg        string
+	outFile    string
+	moduleDirs stringSliceFlag
+	importDirs stringSliceFlag
 )
 
 func init() {
 	flag.StringVar(&license, "license", "ASL2", "License header for generated file (ASL2 or Elastic).")
 	flag.StringVar(&pkg, "pkg", "include", "Package name.")
 	flag.StringVar(&outFile, "out", "include/list.go", "Output file.")
+	flag.Var(&moduleDirs, "moduleDir", "Directory to search for modules to include")
+	flag.Var(&importDirs, "import", "Directory to include")
 	flag.Usage = usageFlag
 }
 
@@ -62,19 +74,19 @@ func main() {
 		log.Fatalf("Invalid license specifier: %v", err)
 	}
 
-	args := flag.Args()
-	if len(args) != 1 {
-		log.Fatal("module-dir must be passed as an argument.")
+	if len(moduleDirs) == 0 && len(importDirs) == 0 {
+		log.Fatal("At least one -import or -moduleDir must be specified.")
 	}
-	dir := args[0]
 
-	// Find modules and datasets.
-	metaDirs, err := mage.FindFiles(
-		filepath.Join(dir, "*/_meta"),
-		filepath.Join(dir, "*/*/_meta"),
-	)
+	dirs, err := findModuleAndDatasets()
 	if err != nil {
-		log.Fatalf("Failed finding modules and datasets: %v", err)
+		log.Fatal(err)
+	}
+
+	if imports, err := findImports(); err != nil {
+		log.Fatal(err)
+	} else {
+		dirs = append(dirs, imports...)
 	}
 
 	// Get the current directories Go import path.
@@ -85,21 +97,36 @@ func main() {
 
 	// Build import paths.
 	var imports []string
-	for _, metaDir := range metaDirs {
-		importDir := filepath.Dir(metaDir)
-
+	for _, dir := range dirs {
 		// Skip dirs that have no .go files.
-		goFiles, err := filepath.Glob(filepath.Join(importDir, "*.go"))
+		goFiles, err := filepath.Glob(filepath.Join(dir, "*.go"))
 		if err != nil {
 			log.Fatal("Failed checking for .go files in package dir: %v", err)
 		}
 		if len(goFiles) == 0 {
 			continue
 		}
 
-		importDir, err = filepath.Rel(mage.CWD(), filepath.Dir(metaDir))
-		if err != nil {
-			log.Fatal(err)
+		// Skip packages without an init() function because that cannot register
+		// anything as a side-effect of being imported (e.g. filebeat/input/file).
+		var foundInitMethod bool
+		for _, f := range goFiles {
+			if hasInitMethod(f) {
+				foundInitMethod = true
+				break
+			}
+		}
+		if !foundInitMethod {
+			continue
+		}
+
+		importDir := dir
+		if filepath.IsAbs(dir) {
+			// Make it relative to the current package if it's absolute.
+			importDir, err = filepath.Rel(mage.CWD(), dir)
+			if err != nil {
+				log.Fatalf("Failure creating import for dir=%v: %v", dir, err)
+			}
 		}
 
 		imports = append(imports, filepath.ToSlash(
@@ -155,3 +182,61 @@ type Data struct {
 	Package string
 	Imports []string
 }
+
+//stringSliceFlag is a flag type that allows more than one value to be specified.
+type stringSliceFlag []string
+
+func (f *stringSliceFlag) String() string { return strings.Join(*f, ", ") }
+
+func (f *stringSliceFlag) Set(value string) error {
+	*f = append(*f, value)
+	return nil
+}
+
+// findModuleAndDatasets searches the specified moduleDirs for packages that
+// should be imported. They are designated by the presence of a _meta dir.
+func findModuleAndDatasets() ([]string, error) {
+	var dirs []string
+	for _, moduleDir := range moduleDirs {
+		// Find modules and datasets as indicated by the _meta dir.
+		metaDirs, err := mage.FindFiles(
+			filepath.Join(moduleDir, "*/_meta"),
+			filepath.Join(moduleDir, "*/*/_meta"),
+		)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed finding modules and datasets")
+		}
+
+		for _, metaDir := range metaDirs {
+			// Strip off _meta.
+			dirs = append(dirs, filepath.Dir(metaDir))
+		}
+	}
+	return dirs, nil
+}
+
+// findImports expands the given import values in case they contain globs.
+func findImports() ([]string, error) {
+	return mage.FindFiles(importDirs...)
+}
+
+// hasInitMethod returns true if the file contains 'func init()'.
+func hasInitMethod(file string) bool {
+	f, err := os.Open(file)
+	if err != nil {
+		log.Fatalf("failed to read from %v: %v", file, err)
+	}
+	defer f.Close()
+
+	var initSignature = []byte("func init()")
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		if bytes.Contains(scanner.Bytes(), initSignature) {
+			return true
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		log.Fatal("failed scanning %v: %v", file, err)
+	}
+	return false
+}

dev-tools/mage/config.go

@@ -18,16 +18,150 @@
 package mage
 
 import (
+	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 )
 
+// Paths to generated config file templates.
+var (
+	shortTemplate     = filepath.Join("build", BeatName+".yml.tmpl")
+	referenceTemplate = filepath.Join("build", BeatName+".reference.yml.tmpl")
+	dockerTemplate    = filepath.Join("build", BeatName+".docker.yml.tmpl")
+)
+
+// ConfigFileType is a bitset that indicates what types of config files to
+// generate.
+type ConfigFileType uint8
+
+// Config file types.
+const (
+	ShortConfigType ConfigFileType = 1 << iota
+	ReferenceConfigType
+	DockerConfigType
+
+	AllConfigTypes ConfigFileType = 0xFF
+)
+
+// IsShort return true if ShortConfigType is set.
+func (t ConfigFileType) IsShort() bool { return t&ShortConfigType > 0 }
+
+// IsReference return true if ReferenceConfigType is set.
+func (t ConfigFileType) IsReference() bool { return t&ReferenceConfigType > 0 }
+
+// IsDocker return true if DockerConfigType is set.
+func (t ConfigFileType) IsDocker() bool { return t&DockerConfigType > 0 }
+
+// ConfigFileParams defines the files that make up each config file.
+type ConfigFileParams struct {
+	ShortParts     []string // List of files or globs.
+	ReferenceParts []string // List of files or globs.
+	DockerParts    []string // List of files or globs.
+	ExtraVars      map[string]interface{}
+}
+
+// Config generates config files. Set DEV_OS and DEV_ARCH to change the target
+// host for the generated configs. Defaults to linux/amd64.
+func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error {
+	if err := makeConfigTemplates(types, args); err != nil {
+		return errors.Wrap(err, "failed making config templates")
+	}
+
+	params := map[string]interface{}{
+		"GOOS":      EnvOr("DEV_OS", "linux"),
+		"GOARCH":    EnvOr("DEV_ARCH", "amd64"),
+		"Reference": false,
+		"Docker":    false,
+	}
+	for k, v := range args.ExtraVars {
+		params[k] = v
+	}
+
+	// Short
+	if types.IsShort() {
+		file := filepath.Join(targetDir, BeatName+".yml")
+		fmt.Printf(">> Building %v for %v/%v\n", file, params["GOOS"], params["GOARCH"])
+		if err := ExpandFile(shortTemplate, file, params); err != nil {
+			return errors.Wrapf(err, "failed building %v", file)
+		}
+	}
+
+	// Reference
+	if types.IsReference() {
+		file := filepath.Join(targetDir, BeatName+".reference.yml")
+		params["Reference"] = true
+		fmt.Printf(">> Building %v for %v/%v\n", file, params["GOOS"], params["GOARCH"])
+		if err := ExpandFile(referenceTemplate, file, params); err != nil {
+			return errors.Wrapf(err, "failed building %v", file)
+		}
+	}
+
+	// Docker
+	if types.IsDocker() {
+		file := filepath.Join(targetDir, BeatName+".docker.yml")
+		params["Reference"] = false
+		params["Docker"] = true
+		fmt.Printf(">> Building %v for %v/%v\n", file, params["GOOS"], params["GOARCH"])
+		if err := ExpandFile(dockerTemplate, file, params); err != nil {
+			return errors.Wrapf(err, "failed building %v", file)
+		}
+	}
+
+	return nil
+}
+
+func makeConfigTemplates(types ConfigFileType, args ConfigFileParams) error {
+	var err error
+
+	if types.IsShort() {
+		if err = makeConfigTemplate(shortTemplate, 0600, args.ShortParts...); err != nil {
+			return err
+		}
+	}
+
+	if types.IsReference() {
+		if err = makeConfigTemplate(referenceTemplate, 0644, args.ReferenceParts...); err != nil {
+			return err
+		}
+	}
+
+	if types.IsDocker() {
+		if err = makeConfigTemplate(dockerTemplate, 0600, args.DockerParts...); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func makeConfigTemplate(destination string, mode os.FileMode, parts ...string) error {
+	configFiles, err := FindFiles(parts...)
+	if err != nil {
+		return errors.Wrap(err, "failed to find config templates")
+	}
+
+	if IsUpToDate(destination, configFiles...) {
+		return nil
+	}
+
+	log.Println(">> Building", destination)
+	if err = FileConcat(destination, mode, configFiles...); err != nil {
+		return err
+	}
+	if err = FindReplace(destination, regexp.MustCompile("beatname"), "{{.BeatName}}"); err != nil {
+		return err
+	}
+	return FindReplace(destination, regexp.MustCompile("beat-index-prefix"), "{{.BeatIndexPrefix}}")
+}
+
 const moduleConfigTemplate = `
 #==========================  Modules configuration =============================
 {{.BeatName}}.modules:

dev-tools/mage/docs.go

@@ -0,0 +1,56 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"log"
+	"path/filepath"
+
+	"github.com/magefile/mage/sh"
+)
+
+type docsBuilder struct{}
+
+// Docs holds the utilities for building documentation.
+var Docs = docsBuilder{}
+
+// FieldDocs generates docs/fields.asciidoc from the specified fields.yml file.
+func (b docsBuilder) FieldDocs(fieldsYML string) error {
+	// Run the docs_collector.py script.
+	ve, err := PythonVirtualenv()
+	if err != nil {
+		return err
+	}
+
+	python, err := LookVirtualenvPath(ve, "python")
+	if err != nil {
+		return err
+	}
+
+	esBeats, err := ElasticBeatsDir()
+	if err != nil {
+		return err
+	}
+
+	log.Println(">> Generating docs/fields.asciidoc for", BeatName)
+	return sh.Run(python, LibbeatDir("scripts/generate_fields_docs.py"),
+		filepath.Dir(fieldsYML),       // Path to dir containing fields.yml.
+		BeatName,                      // Beat title.
+		esBeats,                       // Path to general beats folder.
+		"--output_path", OSSBeatDir()) // It writes to {output_path}/docs/fields.asciidoc.
+}