From b58563becc80d525abd7822252942ac470bc500d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 24 Oct 2018 19:17:04 +0200 Subject: [PATCH] Packaging of journalbeat (#8702) Journalbeat is going to be built using the new Debian 8 container, because systemd version in Debian 7 is too old (v44 instead of the required v187). Minor changes: * add missing X-Pack folder to journalbeat * do not crosscompile journalbeat due to missing dependencies locally --- .travis.yml | 7 +- Makefile | 2 +- dev-tools/mage/crossbuild.go | 36 ++++++---- dev-tools/mage/godaemon.go | 12 +++- dev-tools/mage/settings.go | 5 +- journalbeat/Makefile | 2 - journalbeat/journalbeat.yml | 8 +++ journalbeat/magefile.go | 121 +++++++++++++++++++++++++++++++-- libbeat/scripts/Makefile | 2 + magefile.go | 3 +- x-pack/journalbeat/cmd/root.go | 17 +++++ x-pack/journalbeat/main.go | 19 ++++++ 12 files changed, 206 insertions(+), 28 deletions(-) create mode 100644 x-pack/journalbeat/cmd/root.go create mode 100644 x-pack/journalbeat/main.go diff --git a/.travis.yml b/.travis.yml index 094cfa34dd9..a230e5bbea0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -152,12 +152,11 @@ addons: apt: update: true packages: - - libc6-dev-i386 - - libpcap-dev - - libsystemd-journal-dev - - libxml2-utils - python-virtualenv + - libpcap-dev - xsltproc + - libxml2-utils + - libsystemd-journal-dev before_install: - python --version diff --git a/Makefile b/Makefile index d434eedbfc5..5d8f008e530 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ BUILD_DIR=$(CURDIR)/build COVERAGE_DIR=$(BUILD_DIR)/coverage -BEATS?=auditbeat filebeat heartbeat metricbeat packetbeat winlogbeat +BEATS?=auditbeat filebeat heartbeat journalbeat metricbeat packetbeat winlogbeat PROJECTS=libbeat $(BEATS) PROJECTS_ENV=libbeat filebeat metricbeat PYTHON_ENV?=$(BUILD_DIR)/python-env diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 3e06fba5a15..c75bb57ec57 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -48,6 +48,9 @@ func init() { // CrossBuildOption defines a option to the CrossBuild target. type CrossBuildOption func(params *crossBuildParams) +// ImageSelectorFunc returns the name of the builder image. +type ImageSelectorFunc func(platform string) (string, error) + // ForPlatforms filters the platforms based on the given expression. func ForPlatforms(expr string) func(params *crossBuildParams) { return func(params *crossBuildParams) { @@ -78,16 +81,24 @@ func Serially() func(params *crossBuildParams) { } } +// ImageSelector returns the name of the selected builder image. +func ImageSelector(f ImageSelectorFunc) func(params *crossBuildParams) { + return func(params *crossBuildParams) { + params.ImageSelector = f + } +} + type crossBuildParams struct { - Platforms BuildPlatformList - Target string - Serial bool - InDir string + Platforms BuildPlatformList + Target string + Serial bool + InDir string + ImageSelector ImageSelectorFunc } // CrossBuild executes a given build target once for each target platform. func CrossBuild(options ...CrossBuildOption) error { - params := crossBuildParams{Platforms: Platforms, Target: defaultCrossBuildTarget} + params := crossBuildParams{Platforms: Platforms, Target: defaultCrossBuildTarget, ImageSelector: crossBuildImage} for _, opt := range options { opt(¶ms) } @@ -111,10 +122,10 @@ func CrossBuild(options ...CrossBuildOption) error { if !buildPlatform.Flags.CanCrossBuild() { return fmt.Errorf("unsupported cross build platform %v", buildPlatform.Name) } - builder := GolangCrossBuilder{buildPlatform.Name, params.Target, params.InDir} + builder := GolangCrossBuilder{buildPlatform.Name, params.Target, params.InDir, params.ImageSelector} if params.Serial { if err := builder.Build(); err != nil { - return errors.Wrapf(err, "failed cross-building target=%v for platform=%v", + return errors.Wrapf(err, "failed cross-building target=%v for platform=%v %v", params.ImageSelector, params.Target, buildPlatform.Name) } } else { @@ -174,15 +185,16 @@ func crossBuildImage(platform string) (string, error) { return "", err } - return beatsCrossBuildImage + ":" + goVersion + "-" + tagSuffix, nil + return BeatsCrossBuildImage + ":" + goVersion + "-" + tagSuffix, nil } // GolangCrossBuilder executes the specified mage target inside of the // associated golang-crossbuild container image for the platform. type GolangCrossBuilder struct { - Platform string - Target string - InDir string + Platform string + Target string + InDir string + ImageSelector ImageSelectorFunc } // Build executes the build inside of Docker. @@ -208,7 +220,7 @@ func (b GolangCrossBuilder) Build() error { } dockerRun := sh.RunCmd("docker", "run") - image, err := crossBuildImage(b.Platform) + image, err := b.ImageSelector(b.Platform) if err != nil { return errors.Wrap(err, "failed to determine golang-crossbuild image tag") } diff --git a/dev-tools/mage/godaemon.go b/dev-tools/mage/godaemon.go index 0f4cb2c0ec1..c856fca6448 100644 --- a/dev-tools/mage/godaemon.go +++ b/dev-tools/mage/godaemon.go @@ -23,6 +23,13 @@ import ( "os" ) +var ( + defaultCrossBuildGoDaemon = []CrossBuildOption{ + ForPlatforms("linux"), + WithTarget("buildGoDaemon"), + } +) + // BuildGoDaemon builds the go-deamon binary. func BuildGoDaemon() error { if GOOS != "linux" { @@ -67,6 +74,7 @@ func BuildGoDaemon() error { // CrossBuildGoDaemon cross-build the go-daemon binary using the // golang-crossbuild environment. -func CrossBuildGoDaemon() error { - return CrossBuild(ForPlatforms("linux"), WithTarget("buildGoDaemon")) +func CrossBuildGoDaemon(options ...CrossBuildOption) error { + opts := append(defaultCrossBuildGoDaemon, options...) + return CrossBuild(opts...) } diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index c9229e58c9d..ffaa54991d4 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -39,8 +39,9 @@ const ( fpmVersion = "1.10.0" // Docker images. See https://github.com/elastic/golang-crossbuild. - beatsFPMImage = "docker.elastic.co/beats-dev/fpm" - beatsCrossBuildImage = "docker.elastic.co/beats-dev/golang-crossbuild" + beatsFPMImage = "docker.elastic.co/beats-dev/fpm" + // BeatsCrossBuildImage is the image used for crossbuilding Beats. + BeatsCrossBuildImage = "docker.elastic.co/beats-dev/golang-crossbuild" elasticBeatsImportPath = "github.com/elastic/beats" ) diff --git a/journalbeat/Makefile b/journalbeat/Makefile index c741612e790..62bf3778d21 100644 --- a/journalbeat/Makefile +++ b/journalbeat/Makefile @@ -3,8 +3,6 @@ BEAT_TITLE=Journalbeat SYSTEM_TESTS=false TEST_ENVIRONMENT=false ES_BEATS?=.. -GOX_FLAGS=-cgo -GOX_OS=linux # Path to the libbeat Makefile -include $(ES_BEATS)/libbeat/scripts/Makefile diff --git a/journalbeat/journalbeat.yml b/journalbeat/journalbeat.yml index 81468062a2d..359042a9289 100644 --- a/journalbeat/journalbeat.yml +++ b/journalbeat/journalbeat.yml @@ -142,6 +142,14 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" +#================================ Procesors ===================================== + +# Configure processors to enhance or manipulate events generated by the beat. + +processors: + - add_host_metadata: ~ + - add_cloud_metadata: ~ + #================================ Logging ===================================== # Sets log level. The default log level is info. diff --git a/journalbeat/magefile.go b/journalbeat/magefile.go index 8dfa8c87754..d8566ab9c74 100644 --- a/journalbeat/magefile.go +++ b/journalbeat/magefile.go @@ -22,10 +22,12 @@ package main import ( "context" "fmt" + "strings" "time" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" + "github.com/pkg/errors" "github.com/elastic/beats/dev-tools/mage" ) @@ -33,9 +35,13 @@ import ( func init() { mage.BeatDescription = "Journalbeat ships systemd journal entries to Elasticsearch or Logstash." - // TODO filter platforms + mage.Platforms = mage.Platforms.Filter("linux !linux/ppc64 !linux/mips64") } +const ( + libsystemdDevPkgName = "libsystemd-dev" +) + // Build builds the Beat binary. func Build() error { return mage.Build(mage.DefaultBuildArgs()) @@ -44,6 +50,9 @@ func Build() error { // GolangCrossBuild build the Beat binary inside of the golang-builder. // Do not use directly, use crossBuild instead. func GolangCrossBuild() error { + if d, ok := deps[mage.Platform.Name]; ok { + mg.Deps(d) + } return mage.GolangCrossBuild(mage.DefaultGolangCrossBuildArgs()) } @@ -54,17 +63,17 @@ func BuildGoDaemon() error { // CrossBuild cross-builds the beat for all target platforms. func CrossBuild() error { - return mage.CrossBuild() + return mage.CrossBuild(mage.ImageSelector(selectImage)) } // CrossBuildXPack cross-builds the beat with XPack for all target platforms. func CrossBuildXPack() error { - return mage.CrossBuildXPack() + return mage.CrossBuildXPack(mage.ImageSelector(selectImage)) } // CrossBuildGoDaemon cross-builds the go-daemon binary using Docker. func CrossBuildGoDaemon() error { - return mage.CrossBuildGoDaemon() + return mage.CrossBuildGoDaemon(mage.ImageSelector(selectImage)) } // Clean cleans all generated files and build artifacts. @@ -80,6 +89,7 @@ func Package() { defer func() { fmt.Println("package ran for", time.Since(start)) }() mage.UseElasticBeatPackaging() + mg.Deps(Update) mg.Deps(CrossBuild, CrossBuildXPack, CrossBuildGoDaemon) mg.SerialDeps(mage.Package, TestPackages) @@ -113,3 +123,106 @@ func GoTestUnit(ctx context.Context) error { func GoTestIntegration(ctx context.Context) error { return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) } + +// ----------------------------------------------------------------------------- +// Customizations specific to Journalbeat. +// - Install required headers on builders for different architectures. + +var ( + deps = map[string]func() error{ + "linux/386": installLinux386, + "linux/amd64": installLinuxAMD64, + "linux/arm64": installLinuxARM64, + "linux/armv5": installLinuxARMLE, + "linux/armv6": installLinuxARMLE, + "linux/armv7": installLinuxARMHF, + "linux/mips": installLinuxMIPS, + "linux/mipsle": installLinuxMIPSLE, + "linux/mips64le": installLinuxMIPS64LE, + "linux/ppc64le": installLinuxPPC64LE, + "linux/s390x": installLinuxS390X, + + // No deb packages + //"linux/ppc64": installLinuxPpc64, + //"linux/mips64": installLinuxMips64, + } +) + +func installLinuxAMD64() error { + return installDependencies(libsystemdDevPkgName, "") +} + +func installLinuxARM64() error { + return installDependencies(libsystemdDevPkgName+":arm64", "arm64") +} + +func installLinuxARMHF() error { + return installDependencies(libsystemdDevPkgName+":armhf", "armhf") +} + +func installLinuxARMLE() error { + return installDependencies(libsystemdDevPkgName+":armel", "armel") +} + +func installLinux386() error { + return installDependencies(libsystemdDevPkgName+":i386", "i386") +} + +func installLinuxMIPS() error { + return installDependencies(libsystemdDevPkgName+":mips", "mips") +} + +func installLinuxMIPS64LE() error { + return installDependencies(libsystemdDevPkgName+":mips64el", "mips64el") +} + +func installLinuxMIPSLE() error { + return installDependencies(libsystemdDevPkgName+":mipsel", "mipsel") +} + +func installLinuxPPC64LE() error { + return installDependencies(libsystemdDevPkgName+":ppc64el", "ppc64el") +} + +func installLinuxS390X() error { + return installDependencies(libsystemdDevPkgName+":s390x", "s390x") +} + +func installDependencies(pkg, arch string) error { + if arch != "" { + err := sh.Run("dpkg", "--add-architecture", arch) + if err != nil { + return errors.Wrap(err, "error while adding architecture") + } + } + + if err := sh.Run("apt-get", "update"); err != nil { + return err + } + + return sh.Run("apt-get", "install", "-y", "--no-install-recommends", pkg) +} + +func selectImage(platform string) (string, error) { + tagSuffix := "main" + + switch { + case strings.HasPrefix(platform, "linux/arm"): + tagSuffix = "arm" + case strings.HasPrefix(platform, "linux/mips"): + tagSuffix = "mips" + case strings.HasPrefix(platform, "linux/ppc"): + tagSuffix = "ppc" + case platform == "linux/s390x": + tagSuffix = "s390x" + case strings.HasPrefix(platform, "linux"): + tagSuffix = "main-debian8" + } + + goVersion, err := mage.GoVersion() + if err != nil { + return "", err + } + + return mage.BeatsCrossBuildImage + ":" + goVersion + "-" + tagSuffix, nil +} diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 79c0310a3db..94a0c76cf14 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -106,9 +106,11 @@ ${BEAT_NAME}.test: $(GOFILES_ALL) .PHONY: crosscompile crosscompile: ## @build Cross-compile beat for the OS'es specified in GOX_OS variable. The binaries are placed in the build/bin directory. crosscompile: $(GOFILES) +ifneq ($(shell [[ $(BEAT_NAME) == journalbeat ]] && echo true ),true) go get github.com/mitchellh/gox mkdir -p ${BUILD_DIR}/bin gox -output="${BUILD_DIR}/bin/{{.Dir}}-{{.OS}}-{{.Arch}}" -os="$(strip $(GOX_OS))" -osarch="$(strip $(GOX_OSARCH))" ${GOX_FLAGS} +endif .PHONY: check check: check-headers python-env prepare-tests ## @build Checks project and source code if everything is according to standard diff --git a/magefile.go b/magefile.go index 2084aca2e48..3fad2778368 100644 --- a/magefile.go +++ b/magefile.go @@ -31,6 +31,7 @@ var ( "auditbeat", "filebeat", "heartbeat", + "journalbeat", "metricbeat", "packetbeat", "winlogbeat", @@ -68,5 +69,5 @@ func PackageBeatDashboards() error { // DumpVariables writes the template variables and values to stdout. func DumpVariables() error { - return mage.DumpVariables() + return mage.DumpVariables() } diff --git a/x-pack/journalbeat/cmd/root.go b/x-pack/journalbeat/cmd/root.go new file mode 100644 index 00000000000..50737997086 --- /dev/null +++ b/x-pack/journalbeat/cmd/root.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "github.com/elastic/beats/journalbeat/cmd" + xpackcmd "github.com/elastic/beats/x-pack/libbeat/cmd" +) + +// RootCmd to handle beats cli +var RootCmd = cmd.RootCmd + +func init() { + xpackcmd.AddXPack(RootCmd, cmd.Name) +} diff --git a/x-pack/journalbeat/main.go b/x-pack/journalbeat/main.go new file mode 100644 index 00000000000..f80646bfc49 --- /dev/null +++ b/x-pack/journalbeat/main.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "os" + + "github.com/elastic/beats/x-pack/journalbeat/cmd" + + _ "github.com/elastic/beats/journalbeat/include" +) + +func main() { + if err := cmd.RootCmd.Execute(); err != nil { + os.Exit(1) + } +}