Skip to content

Commit

Permalink
Add Filebeat modules to packages
Browse files Browse the repository at this point in the history
This adds the Filebeat modules files to packages. They were missed during the refactor.

It also fixes a minor issue where files were not marked as config within .rpm files.
  • Loading branch information
andrewkroh committed Jul 3, 2018
1 parent 11d2a64 commit 047c92d
Show file tree
Hide file tree
Showing 5 changed files with 201 additions and 69 deletions.
60 changes: 0 additions & 60 deletions dev-tools/mage/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -237,66 +237,6 @@ func MustFindReplace(file string, re *regexp.Regexp, repl string) {
}
}

// Copy copies a file or a directory (recursively) and preserves the permissions.
func Copy(src, dest string) error {
info, err := os.Stat(src)
if err != nil {
return errors.Wrapf(err, "failed to stat source file %v", src)
}
return recursiveCopy(src, dest, info)
}

func fileCopy(src, dest string, info os.FileInfo) error {
srcFile, err := os.Open(src)
if err != nil {
return err
}
defer srcFile.Close()

if !info.Mode().IsRegular() {
return errors.Errorf("failed to copy source file because it is not a regular file")
}

destFile, err := os.OpenFile(createDir(dest), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()&os.ModePerm)
if err != nil {
return err
}
defer destFile.Close()

if _, err = io.Copy(destFile, srcFile); err != nil {
return err
}
return destFile.Close()
}

func dirCopy(src, dest string, info os.FileInfo) error {
if err := os.MkdirAll(dest, info.Mode()); err != nil {
return errors.Wrap(err, "failed creating dirs")
}

contents, err := ioutil.ReadDir(src)
if err != nil {
return errors.Wrapf(err, "failed to read dir %v", src)
}

for _, info := range contents {
srcFile := filepath.Join(src, info.Name())
destFile := filepath.Join(dest, info.Name())
if err = recursiveCopy(srcFile, destFile, info); err != nil {
return errors.Wrapf(err, "failed to copy %v to %v", srcFile, destFile)
}
}

return nil
}

func recursiveCopy(src, dest string, info os.FileInfo) error {
if info.IsDir() {
return dirCopy(src, dest, info)
}
return fileCopy(src, dest, info)
}

// DownloadFile downloads the given URL and writes the file to destinationDir.
// The path to the file is returned.
func DownloadFile(url, destinationDir string) (string, error) {
Expand Down
130 changes: 130 additions & 0 deletions dev-tools/mage/copy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package mage

import (
"io"
"io/ioutil"
"os"
"path/filepath"
"regexp"

"github.com/pkg/errors"
)

// Copy copies a file or a directory (recursively) and preserves the permissions.
func Copy(src, dest string) error {
copy := &CopyTask{Source: src, Dest: dest}
return copy.Execute()
}

// CopyTask copies a file or directory (recursively) and preserves the permissions.
type CopyTask struct {
Source string // Source directory or file.
Dest string // Destination directory or file.
Mode os.FileMode // Mode to use for copied files. Defaults to preserve permissions.
DirMode os.FileMode // Mode to use for copied dirs. Defaults to preserve permissions.
Exclude []string // Exclude paths that match these regular expressions.
excludes []*regexp.Regexp // Compiled exclude regexes.
}

// Execute executes the copy and returns an error of there is a failure.
func (t *CopyTask) Execute() error {
if err := t.init(); err != nil {
return errors.Wrap(err, "copy failed")
}

info, err := os.Stat(t.Source)
if err != nil {
return errors.Wrapf(err, "copy failed: cannot stat source file %v", t.Source)
}

return errors.Wrap(t.recursiveCopy(t.Source, t.Dest, info), "copy failed")
}

func (t *CopyTask) init() error {
for _, excl := range t.Exclude {
re, err := regexp.Compile(excl)
if err != nil {
return errors.Wrapf(err, "bad exclude pattern %v", excl)
}
t.excludes = append(t.excludes, re)
}
return nil
}

func (t *CopyTask) isExcluded(src string) bool {
for _, excl := range t.excludes {
if match := excl.MatchString(filepath.ToSlash(src)); match {
return true
}
}
return false
}

func (t *CopyTask) recursiveCopy(src, dest string, info os.FileInfo) error {
if info.IsDir() {
return t.dirCopy(src, dest, info)
}
return t.fileCopy(src, dest, info)
}

func (t *CopyTask) fileCopy(src, dest string, info os.FileInfo) error {
if t.isExcluded(src) {
return nil
}

srcFile, err := os.Open(src)
if err != nil {
return err
}
defer srcFile.Close()

if !info.Mode().IsRegular() {
return errors.Errorf("failed to copy source file because it is not a " +
"regular file")
}

mode := t.Mode
if mode == 0 {
mode = info.Mode()
}
destFile, err := os.OpenFile(createDir(dest),
os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode&os.ModePerm)
if err != nil {
return err
}
defer destFile.Close()

if _, err = io.Copy(destFile, srcFile); err != nil {
return err
}
return destFile.Close()
}

func (t *CopyTask) dirCopy(src, dest string, info os.FileInfo) error {
if t.isExcluded(src) {
return nil
}

mode := t.DirMode
if mode == 0 {
mode = info.Mode()
}
if err := os.MkdirAll(dest, mode&os.ModePerm); err != nil {
return errors.Wrap(err, "failed creating dirs")
}

contents, err := ioutil.ReadDir(src)
if err != nil {
return errors.Wrapf(err, "failed to read dir %v", src)
}

for _, info := range contents {
srcFile := filepath.Join(src, info.Name())
destFile := filepath.Join(dest, info.Name())
if err = t.recursiveCopy(srcFile, destFile, info); err != nil {
return errors.Wrapf(err, "failed to copy %v to %v", srcFile, destFile)
}
}

return nil
}
5 changes: 5 additions & 0 deletions dev-tools/mage/pkgtypes.go
Original file line number Diff line number Diff line change
Expand Up @@ -608,6 +608,11 @@ func runFPM(spec PackageSpec, packageType PackageType) error {
if spec.localPostInstallScript != "" {
args = append(args, "--after-install", spec.localPostInstallScript)
}
for _, pf := range spec.Files {
if pf.Config {
args = append(args, "--config-files", pf.Target)
}
}
args = append(args,
"-p", spec.OutputFile,
inputTar,
Expand Down
9 changes: 1 addition & 8 deletions filebeat/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,6 @@ kibana:
@mkdir -p _meta/kibana.generated
@-cp -pr module/*/_meta/kibana/* _meta/kibana.generated

# Collects all modules files to be packaged in a temporary folder
.PHONY: modules
modules:
@mkdir -p _meta/
@rm -rf _meta/module.generated
@rsync -q -av module/ _meta/module.generated --exclude "_meta" --exclude "*/*/test"

# Collects all module configs
.PHONY: configs
configs: python-env
Expand Down Expand Up @@ -49,7 +42,7 @@ imports: python-env

# Runs all collection steps and updates afterwards
.PHONY: collect
collect: fields kibana modules configs collect-docs imports
collect: fields kibana configs collect-docs imports

# Creates a new module. Requires the params MODULE
.PHONY: create-module
Expand Down
66 changes: 65 additions & 1 deletion filebeat/magefile.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,12 @@ package main

import (
"fmt"
"path/filepath"
"time"

"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"github.com/pkg/errors"

"github.com/elastic/beats/dev-tools/mage"
)
Expand Down Expand Up @@ -72,7 +74,9 @@ func Package() {
defer func() { fmt.Println("package ran for", time.Since(start)) }()

mage.UseElasticBeatPackaging()
mg.Deps(Update)
customizePackaging()

mg.Deps(Update, prepareModulePackaging)
mg.Deps(CrossBuild, CrossBuildGoDaemon)
mg.SerialDeps(mage.Package, TestPackages)
}
Expand All @@ -86,3 +90,63 @@ func TestPackages() error {
func Update() error {
return sh.Run("make", "update")
}

// -----------------------------------------------------------------------------
// Customizations specific to Filebeat.
// - Include modules directory in packages (minus _meta and test files).
// - Include modules.d directory in packages.

var modulesDirGenerated = filepath.Clean("build/packaging/modules")

// customizePackaging modifies the package specs to add the modules and
// modules.d directory.
func customizePackaging() {
var (
moduleTarget = "module"
module = mage.PackageFile{
Mode: 0644,
Source: modulesDirGenerated,
}

modulesDTarget = "modules.d"
modulesD = mage.PackageFile{
Mode: 0644,
Source: "modules.d",
Config: true,
}
)

for _, args := range mage.Packages {
pkgType := args.Types[0]
switch pkgType {
case mage.TarGz, mage.Zip:
args.Spec.Files[moduleTarget] = module
args.Spec.Files[modulesDTarget] = modulesD
case mage.Deb, mage.RPM:
args.Spec.Files["/usr/share/{{.BeatName}}/"+moduleTarget] = module
args.Spec.Files["/etc/{{.BeatName}}/"+modulesDTarget] = modulesD
default:
panic(errors.Errorf("unhandled package type: %v", pkgType))
}
}
}

// prepareModulePackaging copies the module dir to the build dir and excludes
// _meta and test files so that they are not included in packages.
func prepareModulePackaging() error {
if err := sh.Rm(modulesDirGenerated); err != nil {
return err
}

copy := &mage.CopyTask{
Source: "module",
Dest: modulesDirGenerated,
Mode: 0644,
DirMode: 0755,
Exclude: []string{
"/_meta",
"/test",
},
}
return copy.Execute()
}

0 comments on commit 047c92d

Please sign in to comment.