Skip to content

Commit

Permalink
Add Filebeat modules to packages (#7487)
Browse files Browse the repository at this point in the history
This adds the Filebeat modules files to packages. They were missed during the refactor.

It also fixes a minor issue where files were not marked as config within .rpm files.
  • Loading branch information
andrewkroh authored and ruflin committed Jul 3, 2018
1 parent 14da408 commit ddeec6d
Show file tree
Hide file tree
Showing 5 changed files with 201 additions and 69 deletions.
60 changes: 0 additions & 60 deletions dev-tools/mage/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -237,66 +237,6 @@ func MustFindReplace(file string, re *regexp.Regexp, repl string) {
}
}

// Copy copies a file or a directory (recursively) and preserves the permissions.
func Copy(src, dest string) error {
info, err := os.Stat(src)
if err != nil {
return errors.Wrapf(err, "failed to stat source file %v", src)
}
return recursiveCopy(src, dest, info)
}

func fileCopy(src, dest string, info os.FileInfo) error {
srcFile, err := os.Open(src)
if err != nil {
return err
}
defer srcFile.Close()

if !info.Mode().IsRegular() {
return errors.Errorf("failed to copy source file because it is not a regular file")
}

destFile, err := os.OpenFile(createDir(dest), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()&os.ModePerm)
if err != nil {
return err
}
defer destFile.Close()

if _, err = io.Copy(destFile, srcFile); err != nil {
return err
}
return destFile.Close()
}

func dirCopy(src, dest string, info os.FileInfo) error {
if err := os.MkdirAll(dest, info.Mode()); err != nil {
return errors.Wrap(err, "failed creating dirs")
}

contents, err := ioutil.ReadDir(src)
if err != nil {
return errors.Wrapf(err, "failed to read dir %v", src)
}

for _, info := range contents {
srcFile := filepath.Join(src, info.Name())
destFile := filepath.Join(dest, info.Name())
if err = recursiveCopy(srcFile, destFile, info); err != nil {
return errors.Wrapf(err, "failed to copy %v to %v", srcFile, destFile)
}
}

return nil
}

func recursiveCopy(src, dest string, info os.FileInfo) error {
if info.IsDir() {
return dirCopy(src, dest, info)
}
return fileCopy(src, dest, info)
}

// DownloadFile downloads the given URL and writes the file to destinationDir.
// The path to the file is returned.
func DownloadFile(url, destinationDir string) (string, error) {
Expand Down
130 changes: 130 additions & 0 deletions dev-tools/mage/copy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package mage

import (
"io"
"io/ioutil"
"os"
"path/filepath"
"regexp"

"github.com/pkg/errors"
)

// Copy copies a file or a directory (recursively) and preserves the permissions.
func Copy(src, dest string) error {
copy := &CopyTask{Source: src, Dest: dest}
return copy.Execute()
}

// CopyTask copies a file or directory (recursively) and preserves the permissions.
type CopyTask struct {
Source string // Source directory or file.
Dest string // Destination directory or file.
Mode os.FileMode // Mode to use for copied files. Defaults to preserve permissions.
DirMode os.FileMode // Mode to use for copied dirs. Defaults to preserve permissions.
Exclude []string // Exclude paths that match these regular expressions.
excludes []*regexp.Regexp // Compiled exclude regexes.
}

// Execute executes the copy and returns an error of there is a failure.
func (t *CopyTask) Execute() error {
if err := t.init(); err != nil {
return errors.Wrap(err, "copy failed")
}

info, err := os.Stat(t.Source)
if err != nil {
return errors.Wrapf(err, "copy failed: cannot stat source file %v", t.Source)
}

return errors.Wrap(t.recursiveCopy(t.Source, t.Dest, info), "copy failed")
}

func (t *CopyTask) init() error {
for _, excl := range t.Exclude {
re, err := regexp.Compile(excl)
if err != nil {
return errors.Wrapf(err, "bad exclude pattern %v", excl)
}
t.excludes = append(t.excludes, re)
}
return nil
}

func (t *CopyTask) isExcluded(src string) bool {
for _, excl := range t.excludes {
if match := excl.MatchString(filepath.ToSlash(src)); match {
return true
}
}
return false
}

func (t *CopyTask) recursiveCopy(src, dest string, info os.FileInfo) error {
if info.IsDir() {
return t.dirCopy(src, dest, info)
}
return t.fileCopy(src, dest, info)
}

func (t *CopyTask) fileCopy(src, dest string, info os.FileInfo) error {
if t.isExcluded(src) {
return nil
}

srcFile, err := os.Open(src)
if err != nil {
return err
}
defer srcFile.Close()

if !info.Mode().IsRegular() {
return errors.Errorf("failed to copy source file because it is not a " +
"regular file")
}

mode := t.Mode
if mode == 0 {
mode = info.Mode()
}
destFile, err := os.OpenFile(createDir(dest),
os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode&os.ModePerm)
if err != nil {
return err
}
defer destFile.Close()

if _, err = io.Copy(destFile, srcFile); err != nil {
return err
}
return destFile.Close()
}

func (t *CopyTask) dirCopy(src, dest string, info os.FileInfo) error {
if t.isExcluded(src) {
return nil
}

mode := t.DirMode
if mode == 0 {
mode = info.Mode()
}
if err := os.MkdirAll(dest, mode&os.ModePerm); err != nil {
return errors.Wrap(err, "failed creating dirs")
}

contents, err := ioutil.ReadDir(src)
if err != nil {
return errors.Wrapf(err, "failed to read dir %v", src)
}

for _, info := range contents {
srcFile := filepath.Join(src, info.Name())
destFile := filepath.Join(dest, info.Name())
if err = t.recursiveCopy(srcFile, destFile, info); err != nil {
return errors.Wrapf(err, "failed to copy %v to %v", srcFile, destFile)
}
}

return nil
}
5 changes: 5 additions & 0 deletions dev-tools/mage/pkgtypes.go
Original file line number Diff line number Diff line change
Expand Up @@ -608,6 +608,11 @@ func runFPM(spec PackageSpec, packageType PackageType) error {
if spec.localPostInstallScript != "" {
args = append(args, "--after-install", spec.localPostInstallScript)
}
for _, pf := range spec.Files {
if pf.Config {
args = append(args, "--config-files", pf.Target)
}
}
args = append(args,
"-p", spec.OutputFile,
inputTar,
Expand Down
9 changes: 1 addition & 8 deletions filebeat/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,6 @@ kibana:
@mkdir -p _meta/kibana.generated
@-cp -pr module/*/_meta/kibana/* _meta/kibana.generated

# Collects all modules files to be packaged in a temporary folder
.PHONY: modules
modules:
@mkdir -p _meta/
@rm -rf _meta/module.generated
@rsync -q -av module/ _meta/module.generated --exclude "_meta" --exclude "*/*/test"

# Collects all module configs
.PHONY: configs
configs: python-env
Expand Down Expand Up @@ -49,7 +42,7 @@ imports: python-env

# Runs all collection steps and updates afterwards
.PHONY: collect
collect: fields kibana modules configs collect-docs imports
collect: fields kibana configs collect-docs imports

# Creates a new module. Requires the params MODULE
.PHONY: create-module
Expand Down
66 changes: 65 additions & 1 deletion filebeat/magefile.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,12 @@ package main

import (
"fmt"
"path/filepath"
"time"

"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"github.com/pkg/errors"

"github.com/elastic/beats/dev-tools/mage"
)
Expand Down Expand Up @@ -72,7 +74,9 @@ func Package() {
defer func() { fmt.Println("package ran for", time.Since(start)) }()

mage.UseElasticBeatPackaging()
mg.Deps(Update)
customizePackaging()

mg.Deps(Update, prepareModulePackaging)
mg.Deps(CrossBuild, CrossBuildGoDaemon)
mg.SerialDeps(mage.Package, TestPackages)
}
Expand All @@ -86,3 +90,63 @@ func TestPackages() error {
func Update() error {
return sh.Run("make", "update")
}

// -----------------------------------------------------------------------------
// Customizations specific to Filebeat.
// - Include modules directory in packages (minus _meta and test files).
// - Include modules.d directory in packages.

var modulesDirGenerated = filepath.Clean("build/packaging/modules")

// customizePackaging modifies the package specs to add the modules and
// modules.d directory.
func customizePackaging() {
var (
moduleTarget = "module"
module = mage.PackageFile{
Mode: 0644,
Source: modulesDirGenerated,
}

modulesDTarget = "modules.d"
modulesD = mage.PackageFile{
Mode: 0644,
Source: "modules.d",
Config: true,
}
)

for _, args := range mage.Packages {
pkgType := args.Types[0]
switch pkgType {
case mage.TarGz, mage.Zip:
args.Spec.Files[moduleTarget] = module
args.Spec.Files[modulesDTarget] = modulesD
case mage.Deb, mage.RPM:
args.Spec.Files["/usr/share/{{.BeatName}}/"+moduleTarget] = module
args.Spec.Files["/etc/{{.BeatName}}/"+modulesDTarget] = modulesD
default:
panic(errors.Errorf("unhandled package type: %v", pkgType))
}
}
}

// prepareModulePackaging copies the module dir to the build dir and excludes
// _meta and test files so that they are not included in packages.
func prepareModulePackaging() error {
if err := sh.Rm(modulesDirGenerated); err != nil {
return err
}

copy := &mage.CopyTask{
Source: "module",
Dest: modulesDirGenerated,
Mode: 0644,
DirMode: 0755,
Exclude: []string{
"/_meta",
"/test",
},
}
return copy.Execute()
}

0 comments on commit ddeec6d

Please sign in to comment.