Merge pull request #396 from graphaelli/build-version

include build time and revision in version information

Makefile

@@ -11,6 +11,8 @@ PREFIX?=.
 BEATS_VERSION?=master
 NOTICE_FILE=NOTICE.txt
 LICENSE_FILE=LICENSE.txt
+NOW=$(shell date -u '+%Y-%m-%dT%H:%M:%S')
+GOBUILD_FLAGS=-i -ldflags "-s -X $(BEAT_PATH)/version.buildTime=$(NOW) -X $(BEAT_PATH)/version.commit=$(COMMIT_ID)"
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/libbeat/scripts/Makefile

NOTICE.txt

@@ -182,7 +182,7 @@ SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: d9011ec66e774da31ce70a89901143b013926736
+Revision: aea09e2cda3db5fcaeeeede23f0d1eb8862cb950
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------

_beats/libbeat/scripts/Makefile

@@ -12,7 +12,7 @@ BEAT_LICENSE?=ASL 2.0 ## @packaging Software license of the application
 BEAT_VENDOR?=Elastic ## @packaging Name of the vendor of the application
 BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 ES_BEATS?=..## @community_beat Must be set to ./vendor/github.com/elastic/beats. It must always be a relative path.
-GOPACKAGES?=$(shell go list ${BEAT_PATH}/... | grep -v /vendor/)
+GOPACKAGES?=$(shell go list ${BEAT_PATH}/... | grep -v /vendor/ | grep -v /scripts/cmd/ )
 PACKER_TEMPLATES_DIR?=${ES_BEATS}/dev-tools/packer ## @Building Directory of templates that are used by "make package"
 NOTICE_FILE?=../NOTICE.txt
 LICENSE_FILE?=../LICENSE.txt
@@ -26,6 +26,7 @@ export PATH := ./bin:$(PATH)
 GOFILES = $(shell find . -type f -name '*.go')
 GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "*/vendor/*")
 GOFILES_ALL = $(GOFILES) $(shell find $(ES_BEATS) -type f -name '*.go')
+GOPACKAGES_STRESSTESTS=$(shell find . -name '*.go' | xargs awk 'FNR>1 {nextfile} /\+build.*stresstest/ {print FILENAME; nextfile}' | xargs dirname | uniq)
 SHELL=bash
 ES_HOST?="elasticsearch"
 PWD=$(shell pwd)
@@ -35,6 +36,7 @@ COVERAGE_TOOL?=${BEAT_GOPATH}/bin/gotestcover
 COVERAGE_TOOL_REPO?=github.com/elastic/beats/vendor/github.com/pierrre/gotestcover
 TESTIFY_TOOL_REPO?=github.com/elastic/beats/vendor/github.com/stretchr/testify
 LIBCOMPOSE_TOOL_REPO?=github.com/docker/libcompose
+GOBUILD_FLAGS?=-i
 GOIMPORTS=goimports
 GOIMPORTS_REPO?=golang.org/x/tools/cmd/goimports
 GOIMPORTS_LOCAL_PREFIX?=github.com/elastic
@@ -48,6 +50,7 @@ TIMEOUT?= 90
 NOSETESTS_OPTIONS?=--process-timeout=$(TIMEOUT) --with-timer -v --with-xunit --xunit-file=${BUILD_DIR}/TEST-system.xml ## @testing the options to pass when calling nosetests
 TEST_ENVIRONMENT?=false ## @testing if true, "make testsuite" runs integration tests and system tests in a dockerized test environment
 SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run unit tests and system tests
+STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests
 GOX_OS?=linux darwin windows solaris freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile".
 GOX_OSARCH?=!darwin/arm !darwin/arm64 !darwin/386 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile".
 GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile".
@@ -89,7 +92,7 @@ endif
 
 
 ${BEAT_NAME}: $(GOFILES_ALL) ## @build build the beat application
-	go build -i
+	go build $(GOBUILD_FLAGS)
 
 # Create test coverage binary
 ${BEAT_NAME}.test: $(GOFILES_ALL)
@@ -113,6 +116,7 @@ check: python-env ## @build Checks project and source code if everything is acco
 
 .PHONY: fmt
 fmt: python-env ## @build Runs `goimports -l -w` and `autopep8`on the project's source code, modifying any files that do not match its style.
+	@go get $(GOIMPORTS_REPO)
 	@goimports -local ${GOIMPORTS_LOCAL_PREFIX} -l -w ${GOFILES_NOVENDOR}
 	@${FIND} -name *.py -exec ${PYTHON_ENV}/bin/autopep8 --in-place --max-line-length 120  {} \;
 
@@ -193,6 +197,14 @@ fast-system-tests: ## @testing Runs system tests without coverage reports and in
 fast-system-tests: ${BEAT_NAME}.test python-env
 	. ${PYTHON_ENV}/bin/activate; nosetests -w tests/system ${NOSETESTS_OPTIONS}
 
+# Runs the go based stress tests
+.PHONY: stress-tests
+stress-tests: ## @testing Runs the stress tests with race detector enabled
+stress-tests:
+	if [ -n '${GOPACKAGES_STRESSTESTS}' ]; then \
+		go test -race --tags=stresstest -v ${GOPACKAGES_STRESSTESTS}; \
+	fi
+
 # Run benchmark tests
 .PHONY: benchmark-tests
 benchmark-tests: ## @testing Runs benchmarks (NOT YET IMPLEMENTED)
@@ -219,6 +231,10 @@ python-env: ${ES_BEATS}/libbeat/tests/system/requirements.txt
 .PHONY: test
 test: ## @testing Runs unit and system tests without coverage reports
 test: unit
+	if [ $(STRESS_TESTS) = true ]; then \
+		 $(MAKE) stress-tests; \
+	fi
+
 	if [ $(SYSTEM_TESTS) = true ]; then \
 		 $(MAKE) fast-system-tests; \
 	fi
@@ -228,6 +244,10 @@ testsuite: ## @testing Runs all tests and generates the coverage reports
 testsuite: clean update
 	$(MAKE) unit-tests
 
+	if [ $(STRESS_TESTS) = true ]; then \
+		 $(MAKE) stress-tests; \
+	fi
+
 	# Setups environment if TEST_ENVIRONMENT is set to true
 	# Only runs integration tests with test environment
 	if [ $(TEST_ENVIRONMENT) = true ]; then \

_beats/libbeat/scripts/cmd/stress_pipeline/main.go

@@ -0,0 +1,89 @@
+// +build !windows
+
+package main
+
+import (
+	"flag"
+	"log"
+	"net/http"
+	_ "net/http/pprof"
+	"time"
+
+	"github.com/elastic/beats/libbeat/beat"
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/logp"
+	"github.com/elastic/beats/libbeat/paths"
+	"github.com/elastic/beats/libbeat/publisher/pipeline/stress"
+	"github.com/elastic/beats/libbeat/service"
+
+	// import queue types
+	_ "github.com/elastic/beats/libbeat/publisher/queue/memqueue"
+
+	// import outputs
+	_ "github.com/elastic/beats/libbeat/outputs/console"
+	_ "github.com/elastic/beats/libbeat/outputs/elasticsearch"
+	_ "github.com/elastic/beats/libbeat/outputs/fileout"
+	_ "github.com/elastic/beats/libbeat/outputs/logstash"
+)
+
+var (
+	duration   time.Duration // -duration <duration>
+	overwrites = common.SettingFlag(nil, "E", "Configuration overwrite")
+)
+
+type config struct {
+	Path    paths.Path
+	Logging logp.Logging
+}
+
+func main() {
+	if err := run(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func run() error {
+	info := beat.Info{
+		Beat:     "stresser",
+		Version:  "0",
+		Name:     "stresser.test",
+		Hostname: "stresser.test",
+	}
+
+	flag.DurationVar(&duration, "duration", 0, "Test duration (default 0)")
+	flag.Parse()
+
+	files := flag.Args()
+	cfg, err := common.LoadFiles(files...)
+	if err != nil {
+		return err
+	}
+
+	service.BeforeRun()
+	defer service.Cleanup()
+
+	if err := cfg.Merge(overwrites); err != nil {
+		return err
+	}
+
+	config := config{}
+	if err := cfg.Unpack(&config); err != nil {
+		return err
+	}
+
+	if err := paths.InitPaths(&config.Path); err != nil {
+		return err
+	}
+	if err = logp.Init("test", time.Now(), &config.Logging); err != nil {
+		return err
+	}
+	logp.SetStderr()
+
+	return stress.RunTests(info, duration, cfg, nil)
+}
+
+func startHTTP(bind string) {
+	go func() {
+		http.ListenAndServe(bind, nil)
+	}()
+}

_beats/libbeat/tests/system/beat/beat.py

@@ -207,7 +207,10 @@ def render_config_template(self, template_name=None,
 
         kargs["beat"] = self
         output_str = template.render(**kargs)
-        with open(os.path.join(self.working_dir, output), "wb") as f:
+
+        output_path = os.path.join(self.working_dir, output)
+        with open(output_path, "wb") as f:
+            os.chmod(output_path, 0600)
             f.write(output_str.encode('utf8'))
 
     # Returns output as JSON object with flattened fields (. notation)

_beats/script/generate_imports.py

@@ -1,18 +1,17 @@
 import sys
 from os import listdir, getcwd
-from os.path import abspath, isdir, join
+from os.path import abspath, isdir, join, dirname, basename
 from argparse import ArgumentParser
 
 sys.path.append(abspath("scripts"))
 from generate_imports_helper import comment, get_importable_lines
 
 
 import_line_format = "\t_ \"{beat_path}/{module}/{name}\""
-include_list_path = "include/list.go"
 import_template = """/*
 {comment}
 */
-package include
+package {package}
 
 import (
 \t// This list is automatically generated by `make imports`
@@ -21,17 +20,21 @@
 """
 
 
-def generate_and_write_to_file(go_beat_path):
+def generate_and_write_to_file(outfile, go_beat_path):
     imported_beat_lines = get_importable_lines(go_beat_path, import_line_format)
     imported_lines = "\n".join(imported_beat_lines)
-    list_go = import_template.format(imports=imported_lines, comment=comment)
-    with open(include_list_path, "w") as output:
+    package = basename(dirname(outfile))
+    list_go = import_template.format(package=package,
+                                     comment=comment,
+                                     imports=imported_lines)
+    with open(outfile, "w") as output:
         output.write(list_go)
 
 
 if __name__ == "__main__":
     parser = ArgumentParser(description="Generate imports for Beats packages")
+    parser.add_argument("--out", default="include/list.go")
     parser.add_argument("beats_path")
     args = parser.parse_args()
 
-    generate_and_write_to_file(args.beats_path)
+    generate_and_write_to_file(args.out, args.beats_path)

_beats/testing/environments/args.yml

@@ -7,4 +7,4 @@ services:
       args:
         DOWNLOAD_URL: https://snapshots.elastic.co/downloads
         ELASTIC_VERSION: 7.0.0-alpha1-SNAPSHOT
-        CACHE_BUST: 20171108
+        CACHE_BUST: 20171201

cmd/root.go

@@ -1,11 +1,14 @@
 package cmd
 
 import (
+	"fmt"
+
 	"github.com/spf13/pflag"
 
 	"github.com/elastic/apm-server/beater"
-
-	cmd "github.com/elastic/beats/libbeat/cmd"
+	"github.com/elastic/apm-server/version"
+	"github.com/elastic/beats/libbeat/cmd"
+	libbeat "github.com/elastic/beats/libbeat/version"
 )
 
 // Name of the beat (apm-server).
@@ -19,5 +22,6 @@ var RootCmd *cmd.BeatsRootCmd
 
 func init() {
 	var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError)
-	RootCmd = cmd.GenRootCmdWithIndexPrefixWithRunFlags(Name, IdxPattern, "", beater.New, runFlags)
+	version := fmt.Sprintf("%s [%s]", libbeat.GetDefaultVersion(), version.String())
+	RootCmd = cmd.GenRootCmdWithIndexPrefixWithRunFlags(Name, IdxPattern, version, beater.New, runFlags)
 }