From 8b7d5fbd96867912b70b3969e1ab6b567096f4b8 Mon Sep 17 00:00:00 2001 From: Cyrille Verrier Date: Tue, 3 Jan 2017 11:03:14 +0100 Subject: [PATCH] libbeat: allow to extend "make clean" rule (#3255) (#3258) --- libbeat/scripts/Makefile | 120 ++++++++--------- libbeat/scripts/generate_makefile_doc.py | 163 +++++++++++++++++++++++ 2 files changed, 222 insertions(+), 61 deletions(-) create mode 100644 libbeat/scripts/generate_makefile_doc.py diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index ab77cdf71ff..7350841b65d 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -1,14 +1,14 @@ ### VARIABLE SETUP ### ### Application using libbeat may override the following variables in their Makefile -BEATNAME?=libbeat -BEAT_DESCRIPTION?=Sends events to Elasticsearch or Logstash -BEAT_VENDOR?=Elastic -BEAT_LICENSE?=ASL 2.0 -BEAT_DOC_URL?=https://www.elastic.co/guide/en/beats/${BEATNAME}/current/index.html +BEATNAME?=libbeat ## @packaging Name of the application +BEAT_DESCRIPTION?=Sends events to Elasticsearch or Logstash ## @packaging Description of the application +BEAT_VENDOR?=Elastic ## @packaging Name of the vendor of the application +BEAT_LICENSE?=ASL 2.0 ## @packaging Software license of the application +BEAT_DOC_URL?=https://www.elastic.co/guide/en/beats/${BEATNAME}/current/index.html ## @packaging Link to the user documentation of the application BEAT_DIR?=github.com/elastic/beats/${BEATNAME} -ES_BEATS?=.. -GOPACKAGES?=${BEAT_DIR}/... -PACKER_TEMPLATES_DIR?=${ES_BEATS}/dev-tools/packer +ES_BEATS?=..## @community_beat Must be set to ./vendor/github.com/elastic/beats +GOPACKAGES?=${BEAT_DIR}/...## @community_beat Must be set to $(shell glide novendor) +PACKER_TEMPLATES_DIR?=${ES_BEATS}/dev-tools/packer ## @Building Directory of templates that are used by "make package" # Makefile for a custom beat that includes this libbeat/scripts/Makefile: # if glide is used to manage vendor dependencies, @@ -42,28 +42,27 @@ COVERAGE_TOOL=${GOPATH}/bin/gotestcover COVERAGE_TOOL_REPO=github.com/elastic/beats/vendor/github.com/pierrre/gotestcover PROCESSES?= 4 TIMEOUT?= 90 -TEST_ENVIRONMENT?=false -SYSTEM_TESTS?=false -GOX_OS?=linux darwin windows solaris freebsd netbsd openbsd -TESTING_ENVIRONMENT?=snapshot -DOCKER_COMPOSE_PROJECT_NAME?=${BEATNAME}_${TESTING_ENVIRONMENT} +TEST_ENVIRONMENT?=false ## @testing if true, "make testsuite" runs integration tests and system tests in a dockerized test environment +SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run unit tests and system tests +GOX_OS?=linux darwin windows solaris freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile". +TESTING_ENVIRONMENT?=snapshot ## @testing The name of the environment under test +DOCKER_COMPOSE_PROJECT_NAME?=${BEATNAME}_${TESTING_ENVIRONMENT} ## @testing The name of the docker-compose project used by the integration and system tests DOCKER_COMPOSE?=TESTING_ENVIRONMENT=${TESTING_ENVIRONMENT} docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} -f docker-compose.yml -DOCKER_CACHE?=1 # If set to 0, all docker images are created without cache +DOCKER_CACHE?=1 ## @miscellaneous If set to 0, all docker images are created without cache GOPACKAGES_COMMA_SEP=$(subst $(space),$(comma),$(strip ${GOPACKAGES})) PYTHON_ENV?=${BUILD_DIR}/python-env -BUILDID?=$(shell git rev-parse HEAD) +BUILDID?=$(shell git rev-parse HEAD) ## @Building The build ID VIRTUALENV_PARAMS?= INTEGRATION_TESTS?= -CGO?=false +CGO?=false ## @building if true, Build with C Go support # Cross compiling targets -TARGETS?="linux/amd64 linux/386 windows/amd64 windows/386 darwin/amd64" -# Cross compiling targets to be build on debian 6. This only applies if CGO is enabled -TARGETS_OLD?="" -PACKAGES?=${BEATNAME}/deb ${BEATNAME}/rpm ${BEATNAME}/darwin ${BEATNAME}/win ${BEATNAME}/bin -SNAPSHOT?=yes -BEATS_BUILDER_IMAGE?=tudorg/beats-builder -BEATS_BUILDER_DEB6_IMAGE?=tudorg/beats-builder-deb6 +TARGETS?="linux/amd64 linux/386 windows/amd64 windows/386 darwin/amd64" ## @building list of platforms/architecture to be built by "make package" +TARGETS_OLD?="" ## @building list of Debian6 architecture to be built by "make package" when CGO is true +PACKAGES?=${BEATNAME}/deb ${BEATNAME}/rpm ${BEATNAME}/darwin ${BEATNAME}/win ${BEATNAME}/bin ## @Building List of OS to be supported by "make package" +SNAPSHOT?=yes ## @Building If yes, builds a snapshot version +BEATS_BUILDER_IMAGE?=tudorg/beats-builder ## @Building Name of the docker image to use when packaging the application +BEATS_BUILDER_DEB6_IMAGE?=tudorg/beats-builder-deb6 ## @Building Name of the docker image to use when packaging the application for Debian 6 ifeq ($(DOCKER_CACHE),0) DOCKER_NOCACHE=--no-cache @@ -77,49 +76,41 @@ endif ### BUILDING ### -# Builds beat -${BEATNAME}: $(GOFILES_ALL) + +${BEATNAME}: $(GOFILES_ALL) ## @build build the beat application go build # Create test coverage binary ${BEATNAME}.test: $(GOFILES_ALL) go test $(RACE) -c -coverpkg ${GOPACKAGES_COMMA_SEP} -# Cross-compile beat for the OS'es specified in GOX_OS variable. -# The binaries are placed in the build/bin directory. .PHONY: crosscompile +crosscompile: ## @build Cross-compile beat for the OS'es specified in GOX_OS variable. The binaries are placed in the build/bin directory. crosscompile: $(GOFILES) go get github.com/mitchellh/gox mkdir -p ${BUILD_DIR}/bin gox -output="${BUILD_DIR}/bin/{{.Dir}}-{{.OS}}-{{.Arch}}" -os="${GOX_OS}" ${GOX_FLAGS} -# Checks project and source code if everything is according to standard + .PHONY: check -check: +check: ## @build Checks project and source code if everything is according to standard @gofmt -l ${GOFILES_NOVENDOR} | (! grep . -q) || (echo "Code differs from gofmt's style" && false) go vet ${GOPACKAGES} -# Runs gofmt -w on the project's source code, modifying any files that do not -# match its style. .PHONY: fmt -fmt: +fmt: ## @build Runs gofmt -w on the project's source code, modifying any files that do not match its style. gofmt -l -w ${GOFILES_NOVENDOR} -# Runs gofmt -s -w on the project's source code, modifying any files that do not -# match its style. .PHONY: simplify -simplify: +simplify: ## @build Runs gofmt -s -w on the project's source code, modifying any files that do not match its style. gofmt -l -s -w ${GOFILES_NOVENDOR} -# Cleans up directory and source code with gofmt .PHONY: clean -clean:: +clean:: ## @build Cleans up all files generated by the build steps rm -rf build ${BEATNAME} ${BEATNAME}.test ${BEATNAME}.exe ${BEATNAME}.test.exe _meta/fields.generated.yml -# Shortcut for continuous integration -# This should always run before merging. .PHONY: ci -ci: +ci: ## @build Shortcut for continuous integration. This should always run before merging. $(MAKE) $(MAKE) check $(MAKE) testsuite @@ -127,7 +118,6 @@ ci: ### Testing ### # Unless stated otherwise, all tests are always run with coverage reporting enabled. - # Prepration for tests .PHONY: prepare-tests prepare-tests: @@ -135,52 +125,54 @@ prepare-tests: # gotestcover is needed to fetch coverage for multiple packages go get ${COVERAGE_TOOL_REPO} -# Runs the unit tests with coverage -# Race is not enabled for unit tests because tests run much slower. .PHONY: unit-tests +unit-tests: ## @testing Runs the unit tests with coverage. Race is not enabled for unit tests because tests run much slower. unit-tests: prepare-tests $(COVERAGE_TOOL) $(RACE) -coverprofile=${COVERAGE_DIR}/unit.cov ${GOPACKAGES} -# Runs the unit tests without coverage reports. .PHONY: unit -unit: +unit: ## @testing Runs the unit tests without coverage reports. go test $(RACE) ${GOPACKAGES} -# Run integration tests. Unit tests are run as part of the integration tests. .PHONY: integration-tests +integration-tests: ## @testing Run integration tests. Unit tests are run as part of the integration tests. integration-tests: prepare-tests $(COVERAGE_TOOL) -tags=integration $(RACE) -coverprofile=${COVERAGE_DIR}/integration.cov ${GOPACKAGES} -# Runs the integration inside a virtual environment. This can be run on any docker-machine (local, remote) +# .PHONY: integration-tests-environment +integration-tests-environment: ## @testing Runs the integration inside a virtual environment. This can be run on any docker-machine (local, remote) integration-tests-environment: prepare-tests build-image ${DOCKER_COMPOSE} run beat make integration-tests RACE_DETECTOR=$(RACE_DETECTOR) # Runs the system tests .PHONY: system-tests +system-tests: ## @testing Runs the system tests system-tests: ${BEATNAME}.test prepare-tests python-env ${ES_BEATS}/libbeat/dashboards/import_dashboards . ${PYTHON_ENV}/bin/activate; INTEGRATION_TESTS=${INTEGRATION_TESTS} nosetests -w tests/system --process-timeout=$(TIMEOUT) --with-timer python ${ES_BEATS}/dev-tools/aggregate_coverage.py -o ${COVERAGE_DIR}/system.cov ./build/system-tests/run # Runs the system tests .PHONY: system-tests-environment +system-tests-environment: ## @testing Runs the system tests inside a virtual environment. This can be run on any docker-machine (local, remote) system-tests-environment: prepare-tests build-image ${DOCKER_COMPOSE} run -e INTEGRATION_TESTS=1 beat make system-tests -# Runs system tests without coverage reports and in parallel + .PHONY: fast-system-tests +fast-system-tests: ## @testing Runs system tests without coverage reports and in parallel fast-system-tests: ${BEATNAME}.test python-env . ${PYTHON_ENV}/bin/activate; nosetests -w tests/system --processes=$(PROCESSES) --process-timeout=$(TIMEOUT) # Run benchmark tests .PHONY: benchmark-tests -benchmark-tests: +benchmark-tests: ## @testing Runs bechmarks (NOT YET IMPLEMENTED) # No benchmark tests exist so far #go test -bench=. ${GOPACKAGES} # Run load tests .PHONY: load-tests -load-tests: +load-tests: ## @testing Runs load tests . ${PYTHON_ENV}/bin/activate; LOAD_TESTS=1 nosetests -w tests/system --processes=$(PROCESSES) --process-timeout=$(TIMEOUT) -a 'load' # Sets up the virtual python environment @@ -194,15 +186,16 @@ python-env: ${ES_BEATS}/libbeat/tests/system/requirements.txt . ${PYTHON_ENV}/bin/activate && pip install -qUr ${ES_BEATS}/libbeat/tests/system/requirements.txt ; \ fi -# Runs unit and system tests without coverage reports + .PHONY: test +test: ## @testing Runs unit and system tests without coverage reports test: unit if [ $(SYSTEM_TESTS) = true ]; then \ $(MAKE) fast-system-tests; \ fi -# Runs all tests and generates the coverage reports .PHONY: testsuite +testsuite: ## @testing Runs all tests and generates the coverage reports testsuite: clean collect $(MAKE) unit-tests @@ -237,8 +230,9 @@ coverage-report: test ! -s ${COVERAGE_DIR}/system.cov || go tool cover -html=${COVERAGE_DIR}/system.cov -o ${COVERAGE_DIR}/system.html test ! -s ${COVERAGE_DIR}/unit.cov || go tool cover -html=${COVERAGE_DIR}/unit.cov -o ${COVERAGE_DIR}/unit.html -# Update expects the most recent version of libbeat in the GOPATH + .PHONY: update +update: ## @build Update expects the most recent version of libbeat in the GOPATH update: python-env collect # Update config @@ -270,15 +264,13 @@ update: python-env collect mkdir -p $(PWD)/_meta/kibana/index-pattern . ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_index_pattern.py --index '${BEATNAME}-*' --libbeat ${ES_BEATS}/libbeat --beat $(PWD) - -# Builds the documents for the beat .PHONY: docs -docs: +docs: ## @build Builds the documents for the beat sh ${ES_BEATS}/libbeat/scripts/build_docs.sh ${BEATNAME} -# Preview the documents for the beat in the browser + .PHONY: docs-preview -docs-preview: +docs-preview: ## @build Preview the documents for the beat in the browser if [ ! -d "build/docs" ]; then $(MAKE) docs; fi; ${BUILD_DIR}/docs/build_docs.pl --chunk=1 -open chunk=1 -open --doc ${GOPATH}/src/github.com/elastic/beats/${BEATNAME}/docs/index.asciidoc -out ${BUILD_DIR}/html_docs @@ -393,8 +385,8 @@ prepare-package-cgo: package-setup: $(MAKE) -C ${ES_BEATS}/dev-tools/packer deps images -# Create binary packages for the beat. .PHONY: package +package: ## @packaging Create binary packages for the beat. package: package-setup echo "Start building packages for ${BEATNAME}" @@ -436,8 +428,14 @@ fix-permissions: # Change ownership of all files inside /build folder from root/root to current user/group docker run -v ${BUILD_DIR}:/build alpine:3.4 sh -c "chown -R $(shell id -u):$(shell id -g) /build" -set_version: +set_version: ## @packaging VERSION=x.y.z set the version of the beat to x.y.z ${ES_BEATS}/dev-tools/set_version ${VERSION} -get_version: +get_version: ## @packaging get the version of the beat. @${ES_BEATS}/dev-tools/get_version + +help: ## @help Show this help. + @python ${ES_BEATS}/libbeat/scripts/generate_makefile_doc.py $(MAKEFILE_LIST) + +help_variables: ## @help Show Makefile customizable variables. + @python ${ES_BEATS}/libbeat/scripts/generate_makefile_doc.py --variables $(MAKEFILE_LIST) diff --git a/libbeat/scripts/generate_makefile_doc.py b/libbeat/scripts/generate_makefile_doc.py new file mode 100644 index 00000000000..0295512eafb --- /dev/null +++ b/libbeat/scripts/generate_makefile_doc.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python + +""" +This script generates and output a documentation from a list of Makefile files + +Example usage: + + python generate_makefile_doc.py Makefile1 Makefile2 ... +""" + +import argparse +import re + + +# Parse a Makefile target line: +# +# Example 1: +# unit: ## @testing Runs the unit tests without coverage reports. +# name => unit +# varname => None +# category => testing +# doc => Runs the unit tests without coverage reports. +# +# Example 2: +# ${BEATNAME}: $(GOFILES_ALL) ## @build build the beat application +# name => None +# varname => BEATNAME +# category => testing +# doc => Runs the unit tests without coverage reports. +regexp_target_doc = re.compile(r'^((?P(-|_|\w)+)|(\${(?P(-|_|\w)+)}))\s*:.*\#\#+\s*@(?P(\w+))\s+(?P(.*))') + + + +# Parse a Makefile variable assignement: +# +# Example 1: +# BEAT_LICENSE?=ASL 2.0 ## @packaging Software license of the application +# name => BEAT_LICENSE +# default => ASL 2.0 +# category => packaging +# doc => Software license of the application +# +## Example 2: +# BEATNAME?=filebeat +# name => BEATNAME +# default => libbeat +# category => None +# doc => None +# +regexp_var_help = re.compile(r'^(?P(\w)+)\s*(\?)?=\s*(?P([^\#]+))(\s+\#\#+\s*@(?P(\w+))(:)?\s+(?P(.*))|\s*$)') + + +# Parse a Makefile line according to the given regexp +# - insert the dict { name, default, is_variable, category, doc} to the categories dictionary +# - insert the category to the categories_set +# - return a pair [name, value] if the line is a Makefile variable assignement +def parse_line(line, regexp, categories, categories_set): + matches = regexp.match(line) + variable = None + if matches: + name = None + variable = False + try: + name = matches.group("varname") + is_variable = True + except: + pass + try: + default = matches.group("default").strip() + except: + default = "" + + if not name: + name = matches.group("name") + is_variable = False + + if name: + variable = [name, default] + + category = matches.group("category") + if category: + category = category.replace("_", " ").capitalize() + doc = matches.group("doc").rstrip('.').rstrip() + doc = doc[0].capitalize() + doc[1:] # Capitalize the first word + + if category not in categories_set: + categories_set.append(category) + categories[category] = [] + + categories[category].append({ + "name": name, + "doc": doc, + "is_variable": is_variable, + "default": default, + }) + return variable + + +# Substitute all Makefile targets whose names are Makefile variables by their final name. +# +# Example in Makefile: +# +# ${BEATNAME}: $(GOFILES_ALL) ## @build build the beat application +# go build +# +# BEATNAME is a Makefile target whose name ${BEATNAME} is a Makefile variable. +# The name of the rule is changed from "BEATNAME" to "filebeat" +# +def substitute_variable_targets(targets, variables): + target_variables = ([target for category in targets for target in targets[category] if target['is_variable']]) + for variable in target_variables: + variable['name'] = variables[variable['name']] + variable['variable'] = False + +# Display the help to stdout +def print_help(categories, categories_set): + column_size = max(len(rule["name"]) for category in categories_set for rule in categories[category]) + for category in categories_set: + print ("\n{}:".format(category)) + for rule in categories[category]: + if "name" in rule: + name = rule["name"] + if "varname" in rule: + name = rule["varname"] + default = rule["default"] + print ("\t{target: <{fill}}\t{doc}.{default}".format( + target=rule["name"], fill=column_size, + doc=rule["doc"], + default=(" Default: {}".format(default) if default else ""))) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generate documentation from a list of Makefile files") + + parser.add_argument( "--variables", dest='variables', + action='store_true') + + parser.add_argument( "files", nargs="+", type=argparse.FileType('r'), + help="list of Makefiles to analyze", + default=None) + args = parser.parse_args() + + categories_targets = {} + categories_vars = {} + categories_targets_set = [] + categories_vars_set = [] + variables = {} + + for file in args.files: + for line in file.readlines(): + parse_line(line, regexp_target_doc, categories_targets, categories_targets_set) + variable = parse_line(line, regexp_var_help, categories_vars, categories_vars_set) + if variable and variable[0] not in variables: + variables[variable[0]] = variable[1] + + substitute_variable_targets(categories_targets, variables) + + if not args.variables: + print ("Usage: make [target] [VARIABLE=value]") + print_help(categories_targets, categories_targets_set) + else: + print_help(categories_vars, categories_vars_set) +