Fix generator jobs for go modules (elastic#16288)

## What does this PR do?

* Set the correct path to `mage` for generator jobs
* Generated beats use go modules for dependency management
   * New option `beats_revision` is added so users can select which beats revision they want to vendor
    * if one does not want to depend on `elastic/beats`, he/she needs to add the appropriate `replace` directive to the generated `go.mod`

## Why is it important?

As Beats is moving to go modules for dependency management, generated Beats should use that as well.

Documentation is provided to help users migrate to go modules in their own Beats. Also, about the new option.

.travis.yml

@@ -148,24 +148,23 @@ jobs:
       stage: test
 
     # Generators
-    # Temporarily disable generator jobs
-    #- os: linux
-    #  env: TARGETS="-C generator/_templates/metricbeat test test-package"
-    #  go: $TRAVIS_GO_VERSION
-    #  stage: test
-    #- os: linux
-    #  env: TARGETS="-C generator/_templates/beat test test-package"
-    #  go: $TRAVIS_GO_VERSION
-    #  stage: test
+    - os: linux
+      env: TARGETS="-C generator/_templates/metricbeat test test-package"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      env: TARGETS="-C generator/_templates/beat test test-package"
+      go: $TRAVIS_GO_VERSION
+      stage: test
 
-    #- os: osx
-    #  env: TARGETS="-C generator/_templates/metricbeat test"
-    #  go: $TRAVIS_GO_VERSION
-    #  stage: test
-    #- os: osx
-    #  env: TARGETS="-C generator/_templates/beat test"
-    #  go: $TRAVIS_GO_VERSION
-    #  stage: test
+    - os: osx
+      env: TARGETS="-C generator/_templates/metricbeat test"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: osx
+      env: TARGETS="-C generator/_templates/beat test"
+      go: $TRAVIS_GO_VERSION
+      stage: test
 
     # Kubernetes
     - os: linux

Jenkinsfile

@@ -459,45 +459,44 @@ pipeline {
             }
           }
         }
-        // Temporarily disable generator jobs
-        //stage('Generators'){
-        //  agent { label 'ubuntu && immutable' }
-        //  options { skipDefaultCheckout() }
-        //  when {
-        //    beforeAgent true
-        //    expression {
-        //      return env.BUILD_GENERATOR != "false"
-        //    }
-        //  }
-        //  stages {
-        //    stage('Generators Metricbeat Linux'){
-        //      steps {
-        //        makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test")
-        //        makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test-package")
-        //      }
-        //    }
-        //    stage('Generators Beat Linux'){
-        //      steps {
-        //        makeTarget("Generators Beat Linux", "-C generator/_templates/beat test")
-        //        makeTarget("Generators Beat Linux", "-C generator/_templates/beat test-package")
-        //      }
-        //    }
-        //    stage('Generators Metricbeat Mac OS X'){
-        //      agent { label 'macosx' }
-        //      options { skipDefaultCheckout() }
-        //      steps {
-        //        makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test")
-        //      }
-        //    }
-        //    stage('Generators Beat Mac OS X'){
-        //      agent { label 'macosx' }
-        //      options { skipDefaultCheckout() }
-        //      steps {
-        //        makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test")
-        //      }
-        //    }
-        //  }
-        //}
+        stage('Generators'){
+          agent { label 'ubuntu && immutable' }
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            expression {
+              return env.BUILD_GENERATOR != "false"
+            }
+          }
+          stages {
+            stage('Generators Metricbeat Linux'){
+              steps {
+                makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test")
+                makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test-package")
+              }
+            }
+            stage('Generators Beat Linux'){
+              steps {
+                makeTarget("Generators Beat Linux", "-C generator/_templates/beat test")
+                makeTarget("Generators Beat Linux", "-C generator/_templates/beat test-package")
+              }
+            }
+            stage('Generators Metricbeat Mac OS X'){
+              agent { label 'macosx' }
+              options { skipDefaultCheckout() }
+              steps {
+                makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test")
+              }
+            }
+            stage('Generators Beat Mac OS X'){
+              agent { label 'macosx' }
+              options { skipDefaultCheckout() }
+              steps {
+                makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test")
+              }
+            }
+          }
+        }
         stage('Kubernetes'){
           agent { label 'ubuntu && immutable' }
           options { skipDefaultCheckout() }
@@ -673,7 +672,7 @@ def loadConfigEnvVars(){
     "^x-pack/functionbeat/.*",
     "^x-pack/libbeat/.*",
   ])
-  //env.BUILD_GENERATOR = isChanged(["^generator/.*"])
+  env.BUILD_GENERATOR = isChanged(["^generator/.*"])
   env.BUILD_HEARTBEAT = isChanged(["^heartbeat/.*"])
   env.BUILD_HEARTBEAT_XPACK = isChanged([
     "^heartbeat/.*",

dev-tools/mage/gomod.go

@@ -18,7 +18,6 @@
 package mage
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 
@@ -80,7 +79,6 @@ func Vendor() error {
 		if err != nil {
 			return err
 		}
-		fmt.Println(path)
 
 		for _, f := range p.filesToCopy {
 			from := filepath.Join(path, f)

dev-tools/mage/gotool/get.go

@@ -27,5 +27,6 @@ func runGoGet(opts ...ArgOpt) error {
 	return runVGo("get", args)
 }
 
+func (goGet) Download() ArgOpt          { return flagBoolIf("-d", true) }
 func (goGet) Update() ArgOpt            { return flagBoolIf("-u", true) }
 func (goGet) Package(pkg string) ArgOpt { return posArg(pkg) }

dev-tools/mage/gotool/go.go

@@ -105,7 +105,7 @@ func ListModulePath(pkg string) (string, error) {
 		return "", err
 	}
 	if n := len(lines); n != 1 {
-		return "", fmt.Errorf("expected 1 line, got %d", n)
+		return "", fmt.Errorf("expected 1 line, got %d while looking for %s", n, pkg)
 	}
 	return lines[0], nil
 }

dev-tools/mage/gotool/modules.go

@@ -19,6 +19,7 @@ package gotool
 
 // Mod is the command go mod.
 var Mod = goMod{
+	Init:   modCommand{"init"}.run,
 	Tidy:   modCommand{"tidy"}.run,
 	Verify: modCommand{"verify"}.run,
 	Vendor: modCommand{"vendor"}.run,
@@ -39,11 +40,15 @@ func (cmd modCommand) run(opts ...ArgOpt) error {
 }
 
 type goMod struct {
+	Init   modInit
 	Tidy   modTidy
 	Verify modVerify
 	Vendor modVendor
 }
 
+// modInit initializes a new go module in folder.
+type modInit func(opts ...ArgOpt) error
+
 // modTidy cleans the go.mod file
 type modTidy func(opts ...ArgOpt) error
 

dev-tools/mage/settings.go

@@ -23,9 +23,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"path"
 	"path/filepath"
-	"reflect"
 	"regexp"
 	"strconv"
 	"strings"
@@ -275,20 +273,15 @@ func ElasticBeatsDir() (string, error) {
 // it will return the root directory of the module from within the module cache or vendor
 // directory.
 func findElasticBeatsDir() (string, error) {
-	// Find the import path for the package containing this file.
-	type foo struct{}
-	typ := reflect.TypeOf(foo{})
-	magepkgpath := typ.PkgPath()
-
-	// Walk up the import path until we find the elastic/beats module path.
-	pkgpath := magepkgpath
-	for extractCanonicalRootImportPath(pkgpath) != elasticBeatsImportPath {
-		pkgpath = path.Dir(pkgpath)
-		if pkgpath == "." {
-			return "", errors.Errorf("failed to find %q from %q", elasticBeatsImportPath, magepkgpath)
-		}
+	repo, err := GetProjectRepoInfo()
+	if err != nil {
+		return "", err
 	}
-	return gotool.ListModulePath(pkgpath)
+	if repo.IsElasticBeats() {
+		return repo.RootDir, nil
+	}
+
+	return gotool.ListModulePath("github.com/elastic/beats/v7")
 }
 
 var (

docs/devguide/newbeat.asciidoc

@@ -26,6 +26,10 @@ The following topics describe how to build a new Beat:
 * <<newbeat-sharing>>
 * <<event-conventions>>
 
+The following topic provide information about moving generated Beats to go modules:
+
+* <<newbeat-migrate-gomodules>>
+
 [[newbeat-getting-ready]]
 === Getting Ready
 
@@ -128,18 +132,21 @@ mage GenerateCustomBeat
 --------------------
 
 The mage script will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`.
-For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname.
+For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname. Finally, pick the revision of elastic/beats you would like to depend on.
 
 [source,shell]
 ---------
 Enter a project name [examplebeat]:  Countbeat
 Enter a github name [your-github-name]: {username}
 Enter a beat path [github.com/{username}/countbeat]: 
 Enter a full name [Firstname Lastname]: {Full Name}
+Enter the github.com/elastic/beats revision [master]:
 ---------
 
 The Beat generator creates a directory called `countbeat` inside of your project folder (e.g. {project folder}/github.com/{github id}/countbeat).
 
+Please note that it is not possible to use revisions of Beats which does not support go modules.
+
 You now have a raw template of the Beat, but you still need to <<setting-up-beat,fetch dependencies and set up the Beat>>.
 
 [[setting-up-beat]]
@@ -482,3 +489,35 @@ func main() {
 
 When you're done with your new Beat, how about letting everyone know? Open
 a pull request to add your link to the {beats-ref}/community-beats.html[Community Beats] list.
+
+[[newbeat-migrate-gomodules]]
+=== Migrate existing Beat to go modules
+
+Get started by making sure the contents of the vendored beats folder is
+pushed to a remote repository. Otherwise, go will overwrite the folder
+with the selected revision of `github.com/elastic/beats`.
+
+The following command will initialize a new module in your repo.
+
+[source,shell]
+----------------------------------------------------------------------
+go mod init
+----------------------------------------------------------------------
+
+Make sure to add the approprite `replace` directives from the `go.mod` file
+of Beats: https://github.com/elastic/beats/tree/master/go.mod
+Unfortunately, this workaround is needed to make sure your Beat can be compiled.
+
+To depend on the latest `master` of `github.com/elastic/beats` run the following command:
+
+[source,shell]
+----------------------------------------------------------------------
+go get github.com/elastic/beats@master
+----------------------------------------------------------------------
+
+To move the dependencies to vendor, you need to manually fetch the new
+`magefile.go` for newly generated Beats from the dev-tools of `elastic/beats`.
+
+We suggest you read the following section to learn about maintaining dependencies
+using go modules:
+* https://github.com/golang/go/wiki/Modules#how-to-upgrade-and-downgrade-dependencies[How to upgrade and downgrade dependencies]

generator/Makefile

@@ -1,11 +1,11 @@
 # Runs test build for the count metricset
 .PHONY: test
 test:
-	$(MAKE) -C beat test
-	$(MAKE) -C metricbeat test
+	$(MAKE) -C _templates/beat test
+	$(MAKE) -C _templates/metricbeat test
 
 # Cleans up environment
 .PHONY: clean
 clean:
-	@$(MAKE) -C beat clean
-	@$(MAKE) -C metricbeat clean
+	@$(MAKE) -C _templates/beat clean
+	@$(MAKE) -C _templates/metricbeat clean

generator/_templates/beat/{beat}/Makefile

@@ -3,11 +3,12 @@ BEAT_PATH={beat_path}
 BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
-ES_BEATS?=./vendor/github.com/elastic/beats
+ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
+ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
 LIBBEAT_MAKEFILE=$(ES_BEATS)/libbeat/scripts/Makefile
-GOPACKAGES=$(shell govendor list -no-status +local)
-GOBUILD_FLAGS=-i -ldflags "-X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)"
-MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
+GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
+GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"
+MAGE_IMPORT_PATH=github.com/magefile/mage
 NO_COLLECT=true
 CHECK_HEADERS_DISABLED=true
 
@@ -16,4 +17,4 @@ CHECK_HEADERS_DISABLED=true
 
 .PHONY: copy-vendor
 copy-vendor:
-	mage vendorUpdate
+	mage vendorUpdate

generator/_templates/beat/{beat}/_meta/beat.reference.yml

@@ -0,0 +1,7 @@
+################### {Beat} Configuration Example #########################
+
+############################# {Beat} ######################################
+
+{beat}:
+  # Defines how often an event is sent to the output
+  period: 1s

generator/_templates/beat/{beat}/include/include.go

@@ -0,0 +1 @@
+package include

generator/_templates/beat/{beat}/magefile.go

@@ -25,7 +25,7 @@ func init() {
 	devtools.BeatProjectType = devtools.CommunityProject
 }
 
-// VendorUpdate updates elastic/beats/v7 in the vendor dir
+// VendorUpdate updates the vendor dir
 func VendorUpdate() error {
 	return beatgen.VendorUpdate()
 }

generator/_templates/beat/{beat}/tools/tools.go

@@ -0,0 +1,14 @@
+// +build tools
+
+// This package contains the tool dependencies of the project.
+
+package tools
+
+import (
+	_ "github.com/pierrre/gotestcover"
+	_ "golang.org/x/tools/cmd/goimports"
+
+	_ "github.com/mitchellh/gox"
+	_ "github.com/reviewdog/reviewdog/cmd/reviewdog"
+	_ "golang.org/x/lint/golint"
+)

generator/_templates/metricbeat/{beat}/Makefile

@@ -3,15 +3,16 @@ BEAT_PATH={beat_path}
 BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
-ES_BEATS?=./vendor/github.com/elastic/beats
-GOPACKAGES=$(shell govendor list -no-status +local)
-GOBUILD_FLAGS=-i -ldflags "-X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)"
-MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
+ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
+ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
+GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
+GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"
+MAGE_IMPORT_PATH=github.com/magefile/mage
 CHECK_HEADERS_DISABLED=true
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/metricbeat/Makefile
 
 .PHONY: copy-vendor
 copy-vendor:
-	mage vendorUpdate
+	mage vendorUpdate