From 047c92dd6b493e4e89f1f6abcf5d099598e52f0c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 2 Jul 2018 22:56:32 -0400 Subject: [PATCH] Add Filebeat modules to packages This adds the Filebeat modules files to packages. They were missed during the refactor. It also fixes a minor issue where files were not marked as config within .rpm files. --- dev-tools/mage/common.go | 60 ----------------- dev-tools/mage/copy.go | 130 +++++++++++++++++++++++++++++++++++++ dev-tools/mage/pkgtypes.go | 5 ++ filebeat/Makefile | 9 +-- filebeat/magefile.go | 66 ++++++++++++++++++- 5 files changed, 201 insertions(+), 69 deletions(-) create mode 100644 dev-tools/mage/copy.go diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index 1ab47d6b5de..4baef2cb4a3 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -237,66 +237,6 @@ func MustFindReplace(file string, re *regexp.Regexp, repl string) { } } -// Copy copies a file or a directory (recursively) and preserves the permissions. -func Copy(src, dest string) error { - info, err := os.Stat(src) - if err != nil { - return errors.Wrapf(err, "failed to stat source file %v", src) - } - return recursiveCopy(src, dest, info) -} - -func fileCopy(src, dest string, info os.FileInfo) error { - srcFile, err := os.Open(src) - if err != nil { - return err - } - defer srcFile.Close() - - if !info.Mode().IsRegular() { - return errors.Errorf("failed to copy source file because it is not a regular file") - } - - destFile, err := os.OpenFile(createDir(dest), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()&os.ModePerm) - if err != nil { - return err - } - defer destFile.Close() - - if _, err = io.Copy(destFile, srcFile); err != nil { - return err - } - return destFile.Close() -} - -func dirCopy(src, dest string, info os.FileInfo) error { - if err := os.MkdirAll(dest, info.Mode()); err != nil { - return errors.Wrap(err, "failed creating dirs") - } - - contents, err := ioutil.ReadDir(src) - if err != nil { - return errors.Wrapf(err, "failed to read dir %v", src) - } - - for _, info := range contents { - srcFile := filepath.Join(src, info.Name()) - destFile := filepath.Join(dest, info.Name()) - if err = recursiveCopy(srcFile, destFile, info); err != nil { - return errors.Wrapf(err, "failed to copy %v to %v", srcFile, destFile) - } - } - - return nil -} - -func recursiveCopy(src, dest string, info os.FileInfo) error { - if info.IsDir() { - return dirCopy(src, dest, info) - } - return fileCopy(src, dest, info) -} - // DownloadFile downloads the given URL and writes the file to destinationDir. // The path to the file is returned. func DownloadFile(url, destinationDir string) (string, error) { diff --git a/dev-tools/mage/copy.go b/dev-tools/mage/copy.go new file mode 100644 index 00000000000..234c31f424f --- /dev/null +++ b/dev-tools/mage/copy.go @@ -0,0 +1,130 @@ +package mage + +import ( + "io" + "io/ioutil" + "os" + "path/filepath" + "regexp" + + "github.com/pkg/errors" +) + +// Copy copies a file or a directory (recursively) and preserves the permissions. +func Copy(src, dest string) error { + copy := &CopyTask{Source: src, Dest: dest} + return copy.Execute() +} + +// CopyTask copies a file or directory (recursively) and preserves the permissions. +type CopyTask struct { + Source string // Source directory or file. + Dest string // Destination directory or file. + Mode os.FileMode // Mode to use for copied files. Defaults to preserve permissions. + DirMode os.FileMode // Mode to use for copied dirs. Defaults to preserve permissions. + Exclude []string // Exclude paths that match these regular expressions. + excludes []*regexp.Regexp // Compiled exclude regexes. +} + +// Execute executes the copy and returns an error of there is a failure. +func (t *CopyTask) Execute() error { + if err := t.init(); err != nil { + return errors.Wrap(err, "copy failed") + } + + info, err := os.Stat(t.Source) + if err != nil { + return errors.Wrapf(err, "copy failed: cannot stat source file %v", t.Source) + } + + return errors.Wrap(t.recursiveCopy(t.Source, t.Dest, info), "copy failed") +} + +func (t *CopyTask) init() error { + for _, excl := range t.Exclude { + re, err := regexp.Compile(excl) + if err != nil { + return errors.Wrapf(err, "bad exclude pattern %v", excl) + } + t.excludes = append(t.excludes, re) + } + return nil +} + +func (t *CopyTask) isExcluded(src string) bool { + for _, excl := range t.excludes { + if match := excl.MatchString(filepath.ToSlash(src)); match { + return true + } + } + return false +} + +func (t *CopyTask) recursiveCopy(src, dest string, info os.FileInfo) error { + if info.IsDir() { + return t.dirCopy(src, dest, info) + } + return t.fileCopy(src, dest, info) +} + +func (t *CopyTask) fileCopy(src, dest string, info os.FileInfo) error { + if t.isExcluded(src) { + return nil + } + + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + if !info.Mode().IsRegular() { + return errors.Errorf("failed to copy source file because it is not a " + + "regular file") + } + + mode := t.Mode + if mode == 0 { + mode = info.Mode() + } + destFile, err := os.OpenFile(createDir(dest), + os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode&os.ModePerm) + if err != nil { + return err + } + defer destFile.Close() + + if _, err = io.Copy(destFile, srcFile); err != nil { + return err + } + return destFile.Close() +} + +func (t *CopyTask) dirCopy(src, dest string, info os.FileInfo) error { + if t.isExcluded(src) { + return nil + } + + mode := t.DirMode + if mode == 0 { + mode = info.Mode() + } + if err := os.MkdirAll(dest, mode&os.ModePerm); err != nil { + return errors.Wrap(err, "failed creating dirs") + } + + contents, err := ioutil.ReadDir(src) + if err != nil { + return errors.Wrapf(err, "failed to read dir %v", src) + } + + for _, info := range contents { + srcFile := filepath.Join(src, info.Name()) + destFile := filepath.Join(dest, info.Name()) + if err = t.recursiveCopy(srcFile, destFile, info); err != nil { + return errors.Wrapf(err, "failed to copy %v to %v", srcFile, destFile) + } + } + + return nil +} diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 4cf15331534..a470ca90663 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -608,6 +608,11 @@ func runFPM(spec PackageSpec, packageType PackageType) error { if spec.localPostInstallScript != "" { args = append(args, "--after-install", spec.localPostInstallScript) } + for _, pf := range spec.Files { + if pf.Config { + args = append(args, "--config-files", pf.Target) + } + } args = append(args, "-p", spec.OutputFile, inputTar, diff --git a/filebeat/Makefile b/filebeat/Makefile index 1c2c6f8d47b..f5d3bf8c6f1 100644 --- a/filebeat/Makefile +++ b/filebeat/Makefile @@ -15,13 +15,6 @@ kibana: @mkdir -p _meta/kibana.generated @-cp -pr module/*/_meta/kibana/* _meta/kibana.generated -# Collects all modules files to be packaged in a temporary folder -.PHONY: modules -modules: - @mkdir -p _meta/ - @rm -rf _meta/module.generated - @rsync -q -av module/ _meta/module.generated --exclude "_meta" --exclude "*/*/test" - # Collects all module configs .PHONY: configs configs: python-env @@ -49,7 +42,7 @@ imports: python-env # Runs all collection steps and updates afterwards .PHONY: collect -collect: fields kibana modules configs collect-docs imports +collect: fields kibana configs collect-docs imports # Creates a new module. Requires the params MODULE .PHONY: create-module diff --git a/filebeat/magefile.go b/filebeat/magefile.go index a5c378e3d99..fcb45b9feaf 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -21,10 +21,12 @@ package main import ( "fmt" + "path/filepath" "time" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" + "github.com/pkg/errors" "github.com/elastic/beats/dev-tools/mage" ) @@ -72,7 +74,9 @@ func Package() { defer func() { fmt.Println("package ran for", time.Since(start)) }() mage.UseElasticBeatPackaging() - mg.Deps(Update) + customizePackaging() + + mg.Deps(Update, prepareModulePackaging) mg.Deps(CrossBuild, CrossBuildGoDaemon) mg.SerialDeps(mage.Package, TestPackages) } @@ -86,3 +90,63 @@ func TestPackages() error { func Update() error { return sh.Run("make", "update") } + +// ----------------------------------------------------------------------------- +// Customizations specific to Filebeat. +// - Include modules directory in packages (minus _meta and test files). +// - Include modules.d directory in packages. + +var modulesDirGenerated = filepath.Clean("build/packaging/modules") + +// customizePackaging modifies the package specs to add the modules and +// modules.d directory. +func customizePackaging() { + var ( + moduleTarget = "module" + module = mage.PackageFile{ + Mode: 0644, + Source: modulesDirGenerated, + } + + modulesDTarget = "modules.d" + modulesD = mage.PackageFile{ + Mode: 0644, + Source: "modules.d", + Config: true, + } + ) + + for _, args := range mage.Packages { + pkgType := args.Types[0] + switch pkgType { + case mage.TarGz, mage.Zip: + args.Spec.Files[moduleTarget] = module + args.Spec.Files[modulesDTarget] = modulesD + case mage.Deb, mage.RPM: + args.Spec.Files["/usr/share/{{.BeatName}}/"+moduleTarget] = module + args.Spec.Files["/etc/{{.BeatName}}/"+modulesDTarget] = modulesD + default: + panic(errors.Errorf("unhandled package type: %v", pkgType)) + } + } +} + +// prepareModulePackaging copies the module dir to the build dir and excludes +// _meta and test files so that they are not included in packages. +func prepareModulePackaging() error { + if err := sh.Rm(modulesDirGenerated); err != nil { + return err + } + + copy := &mage.CopyTask{ + Source: "module", + Dest: modulesDirGenerated, + Mode: 0644, + DirMode: 0755, + Exclude: []string{ + "/_meta", + "/test", + }, + } + return copy.Execute() +}