Generate fields.go for netflow input

dev-tools/cmd/module_include_list/module_include_list.go

@@ -26,6 +26,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"text/template"
 
@@ -133,6 +134,8 @@ func main() {
 			filepath.Join(repo.ImportPath, importDir)))
 	}
 
+	sort.Strings(imports)
+
 	// Populate the template.
 	var buf bytes.Buffer
 	err = Template.Execute(&buf, Data{

dev-tools/mage/modules.go

@@ -0,0 +1,73 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+var modulesDConfigTemplate = `
+# Module: {{.Module}}
+# Docs: https://www.elastic.co/guide/en/beats/{{.BeatName}}/{{ beat_doc_branch }}/{{.BeatName}}-module-{{.Module}}.html
+
+{{.Config}}`[1:]
+
+// GenerateDirModulesD generates a modules.d directory containing the
+// <module>.yml.disabled files. It adds a header to each file containing a
+// link to the documentation.
+func GenerateDirModulesD() error {
+	if err := os.RemoveAll("modules.d"); err != nil {
+		return err
+	}
+
+	shortConfigs, err := filepath.Glob("module/*/_meta/config.yml")
+	if err != nil {
+		return err
+	}
+
+	for _, f := range shortConfigs {
+		parts := strings.Split(filepath.ToSlash(f), "/")
+		if len(parts) < 2 {
+			continue
+		}
+		moduleName := parts[1]
+
+		config, err := ioutil.ReadFile(f)
+		if err != nil {
+			return err
+		}
+
+		data, err := Expand(modulesDConfigTemplate, map[string]interface{}{
+			"Module": moduleName,
+			"Config": string(config),
+		})
+		if err != nil {
+			return err
+		}
+
+		target := filepath.Join("modules.d", moduleName+".yml.disabled")
+		err = ioutil.WriteFile(createDir(target), []byte(data), 0644)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

x-pack/filebeat/input/netflow/_meta/fields.yml

@@ -39,7 +39,7 @@
                 How long the exporter process has been running, in milliseconds.
 
             - name: version
-              type: int
+              type: long
               description: >
                 NetFlow version used.
 

x-pack/filebeat/magefile.go

@@ -9,9 +9,7 @@ package main
 import (
 	"context"
 	"os"
-	"path/filepath"
 	"regexp"
-	"strings"
 
 	"github.com/magefile/mage/mg"
 
@@ -50,9 +48,14 @@ func Clean() error {
 	return mage.Clean()
 }
 
-// Fields generates a fields.yml and fields.go for each module.
+// Fields generates the fields.yml file and a fields.go for each module and
+// input.
 func Fields() {
-	mg.Deps(fieldsYML, moduleFieldsGo)
+	mg.Deps(fieldsYML, moduleFieldsGo, inputFieldsGo)
+}
+
+func inputFieldsGo() error {
+	return mage.GenerateModuleFieldsGo("input")
 }
 
 func moduleFieldsGo() error {
@@ -71,12 +74,16 @@ func Dashboards() error {
 
 // Config generates both the short and reference configs.
 func Config() {
-	mg.Deps(shortConfig, referenceConfig, createDirModulesD)
+	mg.Deps(shortConfig, referenceConfig, mage.GenerateDirModulesD)
 }
 
 // Update is an alias for executing fields, dashboards, config.
 func Update() {
-	mg.SerialDeps(Fields, Dashboards, Config, mage.GenerateModuleIncludeListGo)
+	mg.SerialDeps(Fields, Dashboards, Config, includeList)
+}
+
+func includeList() error {
+	return mage.GenerateIncludeListGo([]string{"input/*"}, []string{"module"})
 }
 
 // Fmt formats source code and adds file headers.
@@ -150,7 +157,7 @@ const (
 // for an x-pack distribution, excluding _meta and test files so that they are
 // not included in packages.
 func prepareModulePackaging() error {
-	mg.Deps(createDirModulesD)
+	mg.Deps(mage.GenerateDirModulesD)
 
 	err := mage.Clean([]string{
 		dirModuleGenerated,
@@ -231,32 +238,3 @@ func referenceConfig() error {
 	mage.MustFindReplace(configFile, regexp.MustCompile("beat-index-prefix"), mage.BeatIndexPrefix)
 	return nil
 }
-
-func createDirModulesD() error {
-	if err := os.RemoveAll("modules.d"); err != nil {
-		return err
-	}
-
-	shortConfigs, err := filepath.Glob("module/*/_meta/config.yml")
-	if err != nil {
-		return err
-	}
-
-	for _, f := range shortConfigs {
-		parts := strings.Split(filepath.ToSlash(f), "/")
-		if len(parts) < 2 {
-			continue
-		}
-		moduleName := parts[1]
-
-		cp := mage.CopyTask{
-			Source: f,
-			Dest:   filepath.Join("modules.d", moduleName+".yml.disabled"),
-			Mode:   0644,
-		}
-		if err = cp.Execute(); err != nil {
-			return err
-		}
-	}
-	return nil
-}

x-pack/filebeat/modules.d/suricata.yml.disabled

@@ -1,3 +1,6 @@
+# Module: suricata
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-suricata.html
+
 - module: suricata
   # All logs
   eve: