diff --git a/.travis.yml b/.travis.yml index 55bf7809561..f875a84ccc2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,10 +74,10 @@ matrix: # Generators - os: linux - env: TARGETS="-C generate/metricbeat test" + env: TARGETS="-C generator/metricbeat test" go: *go_version - os: linux - env: TARGETS="-C generate/beat test" + env: TARGETS="-C generator/beat test" go: *go_version addons: diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 5f04ca4cffa..d568f008d2a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -14,6 +14,8 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] *Affecting all Beats* +- Change beat generator. Use `$GOPATH/src/github.com/elastic/beats/script/generate.py` to generate a beat. {pull}3452[3452] + *Metricbeat* *Packetbeat* diff --git a/Makefile b/Makefile index ffc933f0db2..7535dc0e0c6 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ SNAPSHOT?=yes .PHONY: testsuite testsuite: $(foreach var,$(PROJECTS),$(MAKE) -C $(var) testsuite || exit 1;) - #$(MAKE) -C generate test + #$(MAKE) -C generator test stop-environments: $(foreach var,$(PROJECTS_ENV),$(MAKE) -C $(var) stop-environment || exit 0;) @@ -46,13 +46,13 @@ update: clean: rm -rf build $(foreach var,$(PROJECTS),$(MAKE) -C $(var) clean || exit 1;) - $(MAKE) -C generate clean + $(MAKE) -C generator clean # Cleans up the vendor directory from unnecessary files # This should always be run after updating the dependencies .PHONY: clean-vendor clean-vendor: - sh scripts/clean_vendor.sh + sh script/clean_vendor.sh .PHONY: check check: @@ -118,5 +118,3 @@ upload-release: .PHONY: notice notice: python dev-tools/generate_notice.py . - - diff --git a/generate/beat.py b/generate/beat.py deleted file mode 100644 index 7b75bfe6ad1..00000000000 --- a/generate/beat.py +++ /dev/null @@ -1,7 +0,0 @@ -import helper - -if __name__ == "__main__": - - parser = helper.get_parser() - args = parser.parse_args() - helper.generate_beat("beat", args) diff --git a/generate/metricbeat.py b/generate/metricbeat.py deleted file mode 100644 index b58cbf337e4..00000000000 --- a/generate/metricbeat.py +++ /dev/null @@ -1,7 +0,0 @@ -import helper - -if __name__ == "__main__": - - parser = helper.get_parser() - args = parser.parse_args() - helper.generate_beat("metricbeat", args) diff --git a/generate/metricbeat/Makefile b/generate/metricbeat/Makefile deleted file mode 100644 index 4cbc34dca64..00000000000 --- a/generate/metricbeat/Makefile +++ /dev/null @@ -1,52 +0,0 @@ -BUILD_DIR?=build -PWD=$(shell pwd) -PYTHON_ENV?=${BUILD_DIR}/python-env -BEAT_PATH=${BUILD_DIR}/src/beatpath/testbeat - - - -# Runs test build for mock beat -.PHONY: test -test: python-env - - # Makes sure to use current version of beats for testing - mkdir -p ${BUILD_DIR}/src/github.com/elastic/beats/ - rsync -a --exclude=build ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/ - - mkdir -p ${BEAT_PATH} - export GOPATH=${PWD}/build ; \ - . ${PYTHON_ENV}/bin/activate && python ${PWD}/build/src/github.com/elastic/beats/generate/metricbeat.py --project_name=Testbeat --github_name=ruflin --beat_path=beatpath/testbeat --full_name="Nicolas Ruflin" - - . ${PYTHON_ENV}/bin/activate; \ - export GOPATH=${PWD}/build ; \ - export PATH=${PATH}:${PWD}/build/bin; \ - cd ${BEAT_PATH} ; \ - make copy-vendor ; \ - MODULE=elastic METRICSET=test make create-metricset ; \ - make check ; \ - make update ; \ - make ; \ - make unit - - -# Tests the build process for the beat -.PHONY: test-build -test-build: test - - # Copy dev tools - cp -r ${PWD}/../../../dev-tools ${BEAT_PATH}/vendor/github.com/elastic/beats/ - - cd ${BEAT_PATH}/dev-tools/packer ; \ - make deps ; \ - make images - -# Sets up the virtual python environment -.PHONY: python-env -python-env: - test -d ${PYTHON_ENV} || virtualenv ${PYTHON_ENV} - . ${PYTHON_ENV}/bin/activate && pip install --upgrade pip cookiecutter PyYAML - -# Cleans up environment -.PHONY: clean -clean: - rm -rf build diff --git a/generate/Makefile b/generator/Makefile similarity index 100% rename from generate/Makefile rename to generator/Makefile diff --git a/generate/beat/.gitignore b/generator/beat/.gitignore similarity index 100% rename from generate/beat/.gitignore rename to generator/beat/.gitignore diff --git a/generate/beat/CHANGELOG.md b/generator/beat/CHANGELOG.md similarity index 100% rename from generate/beat/CHANGELOG.md rename to generator/beat/CHANGELOG.md diff --git a/generator/beat/Makefile b/generator/beat/Makefile new file mode 100644 index 00000000000..4a4cc6232d5 --- /dev/null +++ b/generator/beat/Makefile @@ -0,0 +1,3 @@ + + +include ../common/Makefile diff --git a/generate/beat/README.md b/generator/beat/README.md similarity index 100% rename from generate/beat/README.md rename to generator/beat/README.md diff --git a/generate/beat/{beat}/.gitignore b/generator/beat/{beat}/.gitignore similarity index 100% rename from generate/beat/{beat}/.gitignore rename to generator/beat/{beat}/.gitignore diff --git a/generate/beat/{beat}/.travis.yml b/generator/beat/{beat}/.travis.yml similarity index 100% rename from generate/beat/{beat}/.travis.yml rename to generator/beat/{beat}/.travis.yml diff --git a/generate/beat/{beat}/CONTRIBUTING.md b/generator/beat/{beat}/CONTRIBUTING.md similarity index 100% rename from generate/beat/{beat}/CONTRIBUTING.md rename to generator/beat/{beat}/CONTRIBUTING.md diff --git a/generate/beat/{beat}/LICENSE b/generator/beat/{beat}/LICENSE similarity index 100% rename from generate/beat/{beat}/LICENSE rename to generator/beat/{beat}/LICENSE diff --git a/generate/beat/{beat}/Makefile b/generator/beat/{beat}/Makefile similarity index 100% rename from generate/beat/{beat}/Makefile rename to generator/beat/{beat}/Makefile diff --git a/generate/beat/{beat}/NOTICE b/generator/beat/{beat}/NOTICE similarity index 100% rename from generate/beat/{beat}/NOTICE rename to generator/beat/{beat}/NOTICE diff --git a/generate/beat/{beat}/README.md b/generator/beat/{beat}/README.md similarity index 100% rename from generate/beat/{beat}/README.md rename to generator/beat/{beat}/README.md diff --git a/generate/beat/{beat}/_meta/beat.yml b/generator/beat/{beat}/_meta/beat.yml similarity index 100% rename from generate/beat/{beat}/_meta/beat.yml rename to generator/beat/{beat}/_meta/beat.yml diff --git a/generate/beat/{beat}/_meta/fields.yml b/generator/beat/{beat}/_meta/fields.yml similarity index 100% rename from generate/beat/{beat}/_meta/fields.yml rename to generator/beat/{beat}/_meta/fields.yml diff --git a/generate/beat/{beat}/beater/{beat}.go.tmpl b/generator/beat/{beat}/beater/{beat}.go.tmpl similarity index 100% rename from generate/beat/{beat}/beater/{beat}.go.tmpl rename to generator/beat/{beat}/beater/{beat}.go.tmpl diff --git a/generate/beat/{beat}/config/config.go.tmpl b/generator/beat/{beat}/config/config.go.tmpl similarity index 100% rename from generate/beat/{beat}/config/config.go.tmpl rename to generator/beat/{beat}/config/config.go.tmpl diff --git a/generate/beat/{beat}/config/config_test.go.tmpl b/generator/beat/{beat}/config/config_test.go.tmpl similarity index 100% rename from generate/beat/{beat}/config/config_test.go.tmpl rename to generator/beat/{beat}/config/config_test.go.tmpl diff --git a/generate/beat/{beat}/docs/index.asciidoc b/generator/beat/{beat}/docs/index.asciidoc similarity index 100% rename from generate/beat/{beat}/docs/index.asciidoc rename to generator/beat/{beat}/docs/index.asciidoc diff --git a/generate/beat/{beat}/main.go.tmpl b/generator/beat/{beat}/main.go.tmpl similarity index 100% rename from generate/beat/{beat}/main.go.tmpl rename to generator/beat/{beat}/main.go.tmpl diff --git a/generate/beat/{beat}/main_test.go.tmpl b/generator/beat/{beat}/main_test.go.tmpl similarity index 100% rename from generate/beat/{beat}/main_test.go.tmpl rename to generator/beat/{beat}/main_test.go.tmpl diff --git a/generate/beat/{beat}/tests/system/config/{beat}.yml.j2 b/generator/beat/{beat}/tests/system/config/{beat}.yml.j2 similarity index 100% rename from generate/beat/{beat}/tests/system/config/{beat}.yml.j2 rename to generator/beat/{beat}/tests/system/config/{beat}.yml.j2 diff --git a/generate/beat/{beat}/tests/system/requirements.txt b/generator/beat/{beat}/tests/system/requirements.txt similarity index 100% rename from generate/beat/{beat}/tests/system/requirements.txt rename to generator/beat/{beat}/tests/system/requirements.txt diff --git a/generate/beat/{beat}/tests/system/test_base.py b/generator/beat/{beat}/tests/system/test_base.py similarity index 100% rename from generate/beat/{beat}/tests/system/test_base.py rename to generator/beat/{beat}/tests/system/test_base.py diff --git a/generate/beat/{beat}/tests/system/{beat}.py b/generator/beat/{beat}/tests/system/{beat}.py similarity index 100% rename from generate/beat/{beat}/tests/system/{beat}.py rename to generator/beat/{beat}/tests/system/{beat}.py diff --git a/generate/beat/Makefile b/generator/common/Makefile similarity index 79% rename from generate/beat/Makefile rename to generator/common/Makefile index 3d93de44d8e..c4157d9c92d 100644 --- a/generate/beat/Makefile +++ b/generator/common/Makefile @@ -1,31 +1,35 @@ BUILD_DIR?=build PWD=$(shell pwd) -PYTHON_ENV?=${BUILD_DIR}/python-env/ +PYTHON_ENV?=${BUILD_DIR}/python-env +BEAT_TYPE?=beat BEAT_PATH=${BUILD_DIR}/src/beatpath/testbeat ES_BEATS=${GOPATH}/src/github.com/elastic/beats +PREPARE_COMMAND?= # Runs test build for mock beat .PHONY: test -test: python-env - - # Makes sure to use current version of beats for testing - mkdir -p ${BUILD_DIR}/src/github.com/elastic/beats/ - rsync -a --exclude=build ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/ - - mkdir -p ${BEAT_PATH} - export GOPATH=${PWD}/build ; \ - . ${PYTHON_ENV}/bin/activate && python ${PWD}/build/src/github.com/elastic/beats/generate/beat.py --project_name=Testbeat --github_name=ruflin --beat_path=beatpath/testbeat --full_name="Nicolas Ruflin" +test: prepare-test . ${PYTHON_ENV}/bin/activate; \ export GOPATH=${PWD}/build ; \ export PATH=${PATH}:${PWD}/build/bin; \ cd ${BEAT_PATH} ; \ make copy-vendor ; \ + ${PREPARE_COMMAND} \ make check ; \ make update ; \ make ; \ make unit +prepare-test: python-env + # Makes sure to use current version of beats for testing + mkdir -p ${BUILD_DIR}/src/github.com/elastic/beats/ + rsync -a --exclude=build ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/ + + mkdir -p ${BEAT_PATH} + export GOPATH=${PWD}/build ; \ + . ${PYTHON_ENV}/bin/activate && python ${PWD}/build/src/github.com/elastic/beats/script/generate.py --type=${BEAT_TYPE} --project_name=Testbeat --github_name=ruflin --beat_path=beatpath/testbeat --full_name="Nicolas Ruflin" + # Runs test build for the created beat .PHONY: test-build test-build: test diff --git a/generate/metricbeat/.gitignore b/generator/metricbeat/.gitignore similarity index 100% rename from generate/metricbeat/.gitignore rename to generator/metricbeat/.gitignore diff --git a/generate/metricbeat/CHANGELOG.md b/generator/metricbeat/CHANGELOG.md similarity index 100% rename from generate/metricbeat/CHANGELOG.md rename to generator/metricbeat/CHANGELOG.md diff --git a/generate/metricbeat/LICENSE b/generator/metricbeat/LICENSE similarity index 100% rename from generate/metricbeat/LICENSE rename to generator/metricbeat/LICENSE diff --git a/generator/metricbeat/Makefile b/generator/metricbeat/Makefile new file mode 100644 index 00000000000..adc69e2948f --- /dev/null +++ b/generator/metricbeat/Makefile @@ -0,0 +1,5 @@ +BEAT_TYPE=metricbeat +PREPARE_COMMAND=MODULE=elastic METRICSET=test make create-metricset ; + +include ../common/Makefile + diff --git a/generate/metricbeat/README.md b/generator/metricbeat/README.md similarity index 100% rename from generate/metricbeat/README.md rename to generator/metricbeat/README.md diff --git a/generate/metricbeat/{beat}/.gitignore b/generator/metricbeat/{beat}/.gitignore similarity index 100% rename from generate/metricbeat/{beat}/.gitignore rename to generator/metricbeat/{beat}/.gitignore diff --git a/generate/metricbeat/{beat}/CONTRIBUTING.md b/generator/metricbeat/{beat}/CONTRIBUTING.md similarity index 100% rename from generate/metricbeat/{beat}/CONTRIBUTING.md rename to generator/metricbeat/{beat}/CONTRIBUTING.md diff --git a/generate/metricbeat/{beat}/LICENSE b/generator/metricbeat/{beat}/LICENSE similarity index 100% rename from generate/metricbeat/{beat}/LICENSE rename to generator/metricbeat/{beat}/LICENSE diff --git a/generate/metricbeat/{beat}/Makefile b/generator/metricbeat/{beat}/Makefile similarity index 100% rename from generate/metricbeat/{beat}/Makefile rename to generator/metricbeat/{beat}/Makefile diff --git a/generate/metricbeat/{beat}/NOTICE b/generator/metricbeat/{beat}/NOTICE similarity index 100% rename from generate/metricbeat/{beat}/NOTICE rename to generator/metricbeat/{beat}/NOTICE diff --git a/generate/metricbeat/{beat}/README.md b/generator/metricbeat/{beat}/README.md similarity index 100% rename from generate/metricbeat/{beat}/README.md rename to generator/metricbeat/{beat}/README.md diff --git a/generate/metricbeat/{beat}/main.go.tmpl b/generator/metricbeat/{beat}/main.go.tmpl similarity index 100% rename from generate/metricbeat/{beat}/main.go.tmpl rename to generator/metricbeat/{beat}/main.go.tmpl diff --git a/libbeat/docs/newbeat.asciidoc b/libbeat/docs/newbeat.asciidoc index 11126e52a32..85e570b2c42 100644 --- a/libbeat/docs/newbeat.asciidoc +++ b/libbeat/docs/newbeat.asciidoc @@ -142,7 +142,7 @@ Run python and specify the path to the Beat generator: [source,shell] -------------------- -python $GOPATH/src/github.com/elastic/beats/generate/beat.py +python $GOPATH/src/github.com/elastic/beats/script/generate.py -------------------- Python will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`. diff --git a/metricbeat/docs/developer-guide/creating-beat-from-metricbeat.asciidoc b/metricbeat/docs/developer-guide/creating-beat-from-metricbeat.asciidoc index ce9327ff7a5..c35fd6db805 100644 --- a/metricbeat/docs/developer-guide/creating-beat-from-metricbeat.asciidoc +++ b/metricbeat/docs/developer-guide/creating-beat-from-metricbeat.asciidoc @@ -47,7 +47,7 @@ Run the command: [source,bash] ---- -python ${GOPATH}/src/github.com/elastic/beats/generate/metricbeat.py +python ${GOPATH}/src/github.com/elastic/beats/script/generate.py --type=metricbeat ---- When prompted, enter the Beat name and path. diff --git a/scripts/clean_vendor.sh b/script/clean_vendor.sh similarity index 100% rename from scripts/clean_vendor.sh rename to script/clean_vendor.sh diff --git a/generate/helper.py b/script/generate.py similarity index 83% rename from generate/helper.py rename to script/generate.py index 6ea5291cc74..9611011f2c4 100644 --- a/generate/helper.py +++ b/script/generate.py @@ -1,7 +1,7 @@ import os import argparse -# Creates a new beat based on the given parameters +# Creates a new beat or metricbeat based on the given parameters project_name = "" github_name = "" @@ -10,7 +10,8 @@ full_name = "" -def generate_beat(template_path, args): +def generate_beat(args): + global project_name, github_name, beat, beat_path, full_name @@ -27,7 +28,7 @@ def generate_beat(template_path, args): full_name = args.full_name read_input() - process_file(template_path) + process_file(args.type) def read_input(): @@ -50,13 +51,13 @@ def read_input(): full_name = raw_input("Firstname Lastname: ") or "Firstname Lastname" -def process_file(template_path): +def process_file(beat_type): # Load path information - generator_path = os.path.dirname(os.path.realpath(__file__)) + template_path = os.path.dirname(os.path.realpath(__file__)) + '/../generator' go_path = os.environ['GOPATH'] - for root, dirs, files in os.walk(generator_path + '/' + template_path + '/{beat}'): + for root, dirs, files in os.walk(template_path + '/' + beat_type + '/{beat}'): for file in files: @@ -74,7 +75,7 @@ def process_file(template_path): new_path = replace_variables(full_path).replace(".go.tmpl", ".go") # remove generator info and beat name from path - file_path = new_path.replace(generator_path + "/" + template_path + "/" + beat, "") + file_path = new_path.replace(template_path + "/" + beat_type + "/" + beat, "") # New file path to write file content to write_file = go_path + "/src/" + beat_path + "/" + file_path @@ -108,5 +109,16 @@ def get_parser(): parser.add_argument("--github_name", help="Github name") parser.add_argument("--beat_path", help="Beat path") parser.add_argument("--full_name", help="Full name") + parser.add_argument("--type", help="Beat type", default="beat") return parser + + +if __name__ == "__main__": + + parser = get_parser() + args = parser.parse_args() + + generate_beat(args) + +