Add Filebeat modules to packages

dev-tools/mage/common.go

@@ -237,66 +237,6 @@ func MustFindReplace(file string, re *regexp.Regexp, repl string) {
 	}
 }
 
-// Copy copies a file or a directory (recursively) and preserves the permissions.
-func Copy(src, dest string) error {
-	info, err := os.Stat(src)
-	if err != nil {
-		return errors.Wrapf(err, "failed to stat source file %v", src)
-	}
-	return recursiveCopy(src, dest, info)
-}
-
-func fileCopy(src, dest string, info os.FileInfo) error {
-	srcFile, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer srcFile.Close()
-
-	if !info.Mode().IsRegular() {
-		return errors.Errorf("failed to copy source file because it is not a regular file")
-	}
-
-	destFile, err := os.OpenFile(createDir(dest), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()&os.ModePerm)
-	if err != nil {
-		return err
-	}
-	defer destFile.Close()
-
-	if _, err = io.Copy(destFile, srcFile); err != nil {
-		return err
-	}
-	return destFile.Close()
-}
-
-func dirCopy(src, dest string, info os.FileInfo) error {
-	if err := os.MkdirAll(dest, info.Mode()); err != nil {
-		return errors.Wrap(err, "failed creating dirs")
-	}
-
-	contents, err := ioutil.ReadDir(src)
-	if err != nil {
-		return errors.Wrapf(err, "failed to read dir %v", src)
-	}
-
-	for _, info := range contents {
-		srcFile := filepath.Join(src, info.Name())
-		destFile := filepath.Join(dest, info.Name())
-		if err = recursiveCopy(srcFile, destFile, info); err != nil {
-			return errors.Wrapf(err, "failed to copy %v to %v", srcFile, destFile)
-		}
-	}
-
-	return nil
-}
-
-func recursiveCopy(src, dest string, info os.FileInfo) error {
-	if info.IsDir() {
-		return dirCopy(src, dest, info)
-	}
-	return fileCopy(src, dest, info)
-}
-
 // DownloadFile downloads the given URL and writes the file to destinationDir.
 // The path to the file is returned.
 func DownloadFile(url, destinationDir string) (string, error) {

dev-tools/mage/copy.go

@@ -0,0 +1,130 @@
+package mage
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+
+	"github.com/pkg/errors"
+)
+
+// Copy copies a file or a directory (recursively) and preserves the permissions.
+func Copy(src, dest string) error {
+	copy := &CopyTask{Source: src, Dest: dest}
+	return copy.Execute()
+}
+
+// CopyTask copies a file or directory (recursively) and preserves the permissions.
+type CopyTask struct {
+	Source   string           // Source directory or file.
+	Dest     string           // Destination directory or file.
+	Mode     os.FileMode      // Mode to use for copied files. Defaults to preserve permissions.
+	DirMode  os.FileMode      // Mode to use for copied dirs. Defaults to preserve permissions.
+	Exclude  []string         // Exclude paths that match these regular expressions.
+	excludes []*regexp.Regexp // Compiled exclude regexes.
+}
+
+// Execute executes the copy and returns an error of there is a failure.
+func (t *CopyTask) Execute() error {
+	if err := t.init(); err != nil {
+		return errors.Wrap(err, "copy failed")
+	}
+
+	info, err := os.Stat(t.Source)
+	if err != nil {
+		return errors.Wrapf(err, "copy failed: cannot stat source file %v", t.Source)
+	}
+
+	return errors.Wrap(t.recursiveCopy(t.Source, t.Dest, info), "copy failed")
+}
+
+func (t *CopyTask) init() error {
+	for _, excl := range t.Exclude {
+		re, err := regexp.Compile(excl)
+		if err != nil {
+			return errors.Wrapf(err, "bad exclude pattern %v", excl)
+		}
+		t.excludes = append(t.excludes, re)
+	}
+	return nil
+}
+
+func (t *CopyTask) isExcluded(src string) bool {
+	for _, excl := range t.excludes {
+		if match := excl.MatchString(filepath.ToSlash(src)); match {
+			return true
+		}
+	}
+	return false
+}
+
+func (t *CopyTask) recursiveCopy(src, dest string, info os.FileInfo) error {
+	if info.IsDir() {
+		return t.dirCopy(src, dest, info)
+	}
+	return t.fileCopy(src, dest, info)
+}
+
+func (t *CopyTask) fileCopy(src, dest string, info os.FileInfo) error {
+	if t.isExcluded(src) {
+		return nil
+	}
+
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+
+	if !info.Mode().IsRegular() {
+		return errors.Errorf("failed to copy source file because it is not a " +
+			"regular file")
+	}
+
+	mode := t.Mode
+	if mode == 0 {
+		mode = info.Mode()
+	}
+	destFile, err := os.OpenFile(createDir(dest),
+		os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode&os.ModePerm)
+	if err != nil {
+		return err
+	}
+	defer destFile.Close()
+
+	if _, err = io.Copy(destFile, srcFile); err != nil {
+		return err
+	}
+	return destFile.Close()
+}
+
+func (t *CopyTask) dirCopy(src, dest string, info os.FileInfo) error {
+	if t.isExcluded(src) {
+		return nil
+	}
+
+	mode := t.DirMode
+	if mode == 0 {
+		mode = info.Mode()
+	}
+	if err := os.MkdirAll(dest, mode&os.ModePerm); err != nil {
+		return errors.Wrap(err, "failed creating dirs")
+	}
+
+	contents, err := ioutil.ReadDir(src)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read dir %v", src)
+	}
+
+	for _, info := range contents {
+		srcFile := filepath.Join(src, info.Name())
+		destFile := filepath.Join(dest, info.Name())
+		if err = t.recursiveCopy(srcFile, destFile, info); err != nil {
+			return errors.Wrapf(err, "failed to copy %v to %v", srcFile, destFile)
+		}
+	}
+
+	return nil
+}

dev-tools/mage/pkgtypes.go

@@ -608,6 +608,11 @@ func runFPM(spec PackageSpec, packageType PackageType) error {
 	if spec.localPostInstallScript != "" {
 		args = append(args, "--after-install", spec.localPostInstallScript)
 	}
+	for _, pf := range spec.Files {
+		if pf.Config {
+			args = append(args, "--config-files", pf.Target)
+		}
+	}
 	args = append(args,
 		"-p", spec.OutputFile,
 		inputTar,

filebeat/Makefile

@@ -15,13 +15,6 @@ kibana:
 	@mkdir -p _meta/kibana.generated
 	@-cp -pr module/*/_meta/kibana/* _meta/kibana.generated
 
-# Collects all modules files to be packaged in a temporary folder
-.PHONY: modules
-modules:
-	@mkdir -p _meta/
-	@rm -rf _meta/module.generated
-	@rsync -q -av module/ _meta/module.generated --exclude "_meta" --exclude "*/*/test"
-
 # Collects all module configs
 .PHONY: configs
 configs: python-env
@@ -49,7 +42,7 @@ imports: python-env
 
 # Runs all collection steps and updates afterwards
 .PHONY: collect
-collect: fields kibana modules configs collect-docs imports
+collect: fields kibana configs collect-docs imports
 
 # Creates a new module. Requires the params MODULE
 .PHONY: create-module

filebeat/magefile.go

@@ -21,10 +21,12 @@ package main
 
 import (
 	"fmt"
+	"path/filepath"
 	"time"
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
+	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/dev-tools/mage"
 )
@@ -72,7 +74,9 @@ func Package() {
 	defer func() { fmt.Println("package ran for", time.Since(start)) }()
 
 	mage.UseElasticBeatPackaging()
-	mg.Deps(Update)
+	customizePackaging()
+
+	mg.Deps(Update, prepareModulePackaging)
 	mg.Deps(CrossBuild, CrossBuildGoDaemon)
 	mg.SerialDeps(mage.Package, TestPackages)
 }
@@ -86,3 +90,63 @@ func TestPackages() error {
 func Update() error {
 	return sh.Run("make", "update")
 }
+
+// -----------------------------------------------------------------------------
+// Customizations specific to Filebeat.
+// - Include modules directory in packages (minus _meta and test files).
+// - Include modules.d directory in packages.
+
+var modulesDirGenerated = filepath.Clean("build/packaging/modules")
+
+// customizePackaging modifies the package specs to add the modules and
+// modules.d directory.
+func customizePackaging() {
+	var (
+		moduleTarget = "module"
+		module       = mage.PackageFile{
+			Mode:   0644,
+			Source: modulesDirGenerated,
+		}
+
+		modulesDTarget = "modules.d"
+		modulesD       = mage.PackageFile{
+			Mode:   0644,
+			Source: "modules.d",
+			Config: true,
+		}
+	)
+
+	for _, args := range mage.Packages {
+		pkgType := args.Types[0]
+		switch pkgType {
+		case mage.TarGz, mage.Zip:
+			args.Spec.Files[moduleTarget] = module
+			args.Spec.Files[modulesDTarget] = modulesD
+		case mage.Deb, mage.RPM:
+			args.Spec.Files["/usr/share/{{.BeatName}}/"+moduleTarget] = module
+			args.Spec.Files["/etc/{{.BeatName}}/"+modulesDTarget] = modulesD
+		default:
+			panic(errors.Errorf("unhandled package type: %v", pkgType))
+		}
+	}
+}
+
+// prepareModulePackaging copies the module dir to the build dir and excludes
+// _meta and test files so that they are not included in packages.
+func prepareModulePackaging() error {
+	if err := sh.Rm(modulesDirGenerated); err != nil {
+		return err
+	}
+
+	copy := &mage.CopyTask{
+		Source:  "module",
+		Dest:    modulesDirGenerated,
+		Mode:    0644,
+		DirMode: 0755,
+		Exclude: []string{
+			"/_meta",
+			"/test",
+		},
+	}
+	return copy.Execute()
+}