diff --git a/.travis.yml b/.travis.yml index 8c61a23b6fd2..f68c06c233d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,35 +37,56 @@ jobs: # Filebeat - os: linux - env: TARGETS="-C filebeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh filebeat libbeat || travis_terminate 0 + env: + - PROJECT=filebeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test - os: osx - env: TARGETS="TEST_ENVIRONMENT=0 -C filebeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh filebeat libbeat || travis_terminate 0 + env: + - PROJECT=filebeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test - os: linux - env: TARGETS="-C x-pack/filebeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh x-pack/filebeat filebeat libbeat || travis_terminate 0 + env: + - PROJECT=x-pack/filebeat + - MAGE='build test' go: $(GO_VERSION) stage: test # Heartbeat - os: linux - env: TARGETS="-C heartbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh heartbeat libbeat || travis_terminate 0 + env: + - PROJECT=heartbeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test - os: osx - env: TARGETS="TEST_ENVIRONMENT=0 -C heartbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh heartbeat libbeat || travis_terminate 0 + env: + - PROJECT=heartbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test # Auditbeat - os: linux - env: TARGETS="-C auditbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh auditbeat libbeat || travis_terminate 0 + env: + - PROJECT=auditbeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test - os: osx - env: TARGETS="TEST_ENVIRONMENT=0 -C auditbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh auditbeat libbeat || travis_terminate 0 + env: + - PROJECT=auditbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test - os: linux @@ -73,13 +94,28 @@ jobs: go: $TRAVIS_GO_VERSION stage: test - os: linux - env: TARGETS="-C x-pack/auditbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh x-pack/auditbeat auditbeat libbeat || travis_terminate 0 + env: + - PROJECT=x-pack/auditbeat + - MAGE='build test' + go: $TRAVIS_GO_VERSION + stage: test + - os: osx + before_install: .ci/scripts/travis_has_changes.sh x-pack/auditbeat auditbeat libbeat || travis_terminate 0 + env: + - PROJECT=x-pack/auditbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test # Libbeat - os: linux - env: TARGETS="-C libbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh libbeat || travis_terminate 0 + env: + - PROJECT=libbeat + - MAGE='build test' + # The libbeat tests are so verbose that they exceed the maximum allowed log length of Travis CI. + - MAGEFILE_VERBOSE=false go: $TRAVIS_GO_VERSION stage: test - os: linux @@ -91,32 +127,45 @@ jobs: go: $TRAVIS_GO_VERSION stage: test - os: linux - env: TARGETS="-C x-pack/libbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh x-pack/libbeat libbeat || travis_terminate 0 + env: + - PROJECT=x-pack/libbeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test # Metricbeat - os: linux - env: TARGETS="-C metricbeat unit-tests" + before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0 + env: + - PROJECT=metricbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test - os: linux before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0 - install: + install: - .ci/scripts/install-kind.sh - .ci/scripts/install-kubectl.sh env: - - TARGETS="-C metricbeat integration-tests" - K8S_VERSION=v1.17.2 - KIND_VERSION=v0.7.0 + - PROJECT=metricbeat + - MAGE='goIntegTest' go: $TRAVIS_GO_VERSION stage: test - os: linux - env: TARGETS="-C metricbeat system-tests" + before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0 + env: + - PROJECT=metricbeat + - MAGE='pythonIntegTest' go: $TRAVIS_GO_VERSION stage: test - os: osx - env: TARGETS="-C metricbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0 + env: + - PROJECT=metricbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test - os: linux @@ -124,25 +173,40 @@ jobs: go: $TRAVIS_GO_VERSION stage: test - os: linux - env: TARGETS="-C x-pack/metricbeat unit-tests" + before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0 + env: + - PROJECT=x-pack/metricbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test - os: linux - env: TARGETS="-C x-pack/metricbeat integration-tests" + before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0 + env: + - PROJECT=x-pack/metricbeat + - MAGE='goIntegTest' go: $TRAVIS_GO_VERSION stage: test - os: linux - env: TARGETS="-C x-pack/metricbeat system-tests" + before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0 + env: + - PROJECT=x-pack/metricbeat + - MAGE='pythonIntegTest' go: $TRAVIS_GO_VERSION stage: test - os: osx - env: TARGETS="-C x-pack/metricbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0 + env: + - PROJECT=x-pack/metricbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test # Packetbeat - os: linux - env: TARGETS="-C packetbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh packetbeat libbeat || travis_terminate 0 + env: + - PROJECT=packetbeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test @@ -154,40 +218,58 @@ jobs: # Functionbeat - os: linux - env: TARGETS="-C x-pack/functionbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh x-pack/functionbeat libbeat || travis_terminate 0 + env: + - PROJECT=x-pack/functionbeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test - os: osx - env: TARGETS="TEST_ENVIRONMENT=0 -C x-pack/functionbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh x-pack/functionbeat libbeat || travis_terminate 0 + env: + - PROJECT=x-pack/functionbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test - os: linux before_install: .ci/scripts/travis_has_changes.sh x-pack/functionbeat libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/functionbeat test-gcp-functions" + env: + - PROJECT=x-pack/functionbeat + - MAGE='testGCPFunctions' go: 1.13.1 stage: test # Docker Log Driver - os: linux - env: TARGETS="-C x-pack/dockerlogbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh x-pack/dockerlogbeat libbeat || travis_terminate 0 + env: + - PROJECT=x-pack/dockerlogbeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test # Journalbeat - os: linux - env: TARGETS="-C journalbeat testsuite" + before_install: .ci/scripts/travis_has_changes.sh journalbeat libbeat || travis_terminate 0 + env: + - PROJECT=journalbeat + - MAGE='build goUnitTest' go: $TRAVIS_GO_VERSION stage: test # Agent - os: linux before_install: .ci/scripts/travis_has_changes.sh x-pack/elastic-agent libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/elastic-agent testsuite" + env: + - PROJECT=x-pack/elastic-agent + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test - os: osx before_install: .ci/scripts/travis_has_changes.sh x-pack/elastic-agent libbeat || travis_terminate 0 - env: TARGETS="TEST_ENVIRONMENT=0 -C x-pack/elastic-agent testsuite" + env: + - PROJECT=x-pack/elastic-agent + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test @@ -279,6 +361,7 @@ before_install: - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - if [ $TRAVIS_OS_NAME = osx ]; then pip install virtualenv==16.7.9; fi + - make mage # Skips installations step @@ -289,7 +372,14 @@ script: # Default Travis timeout is 10min, so this workaround prints timestamps every 9min to reset the counter. # Using seconds (540s = 9min) instead of minutes for shell compatibility reasons. - while sleep 540; do echo "=====[ ${SECONDS} seconds still running ]====="; done & - - make $TARGETS + - if [[ -n "$MAGE" ]]; then + echo ">> mage $MAGE from $PROJECT"; + cd "$PROJECT"; + mage $MAGE; + else + echo ">> make $TARGETS"; + make $TARGETS; + fi - kill %1 notifications: diff --git a/Jenkinsfile b/Jenkinsfile index 282a30c3c40d..9d5ddd347454 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -144,7 +144,7 @@ pipeline { } } steps { - makeTarget(context: "Filebeat oss Linux", directory: 'filebeat', target: 'testsuite', withModule: true) + mageTarget(context: "Filebeat oss Linux", directory: "filebeat", target: "build test", withModule: true) } } stage('Filebeat x-pack'){ @@ -157,7 +157,7 @@ pipeline { } } steps { - mageTarget(context: "Filebeat x-pack Linux", directory: "x-pack/filebeat", target: "update build test", withModule: true) + mageTarget(context: "Filebeat x-pack Linux", directory: "x-pack/filebeat", target: "build test", withModule: true) } } stage('Filebeat Mac OS X'){ @@ -234,7 +234,7 @@ pipeline { stages { stage('Heartbeat oss'){ steps { - makeTarget(context: "Heartbeat oss Linux", directory: 'heartbeat', target: "testsuite") + mageTarget(context: "Heartbeat oss Linux", directory: "heartbeat", target: "build test") } } stage('Heartbeat Mac OS X'){ @@ -280,7 +280,7 @@ pipeline { } } steps { - makeTarget(context: "Auditbeat oss Linux", directory: 'auditbeat', target: "testsuite", withModule: true) + mageTarget(context: "Auditbeat oss Linux", directory: "auditbeat", target: "build test") } } stage('Auditbeat crosscompile'){ @@ -378,7 +378,7 @@ pipeline { stages { stage('Libbeat oss'){ steps { - makeTarget(context: "Libbeat oss Linux", directory: 'libbeat', target: "testsuite") + mageTarget(context: "Libbeat oss Linux", directory: "libbeat", target: "build test") } } stage('Libbeat crosscompile'){ @@ -403,7 +403,7 @@ pipeline { } } steps { - makeTarget(context: "Libbeat x-pack Linux", directory: 'x-pack/libbeat', target: "testsuite") + mageTarget(context: "Libbeat x-pack Linux", directory: "x-pack/libbeat", target: "build test") } } stage('Metricbeat OSS Unit tests'){ @@ -560,9 +560,40 @@ pipeline { } } stages { - stage('Packetbeat oss'){ + stage('Packetbeat Linux'){ steps { - makeTarget(context: "Packetbeat oss Linux", directory: 'packetbeat', target: "testsuite") + mageTarget(context: "Packetbeat OSS Linux", directory: "packetbeat", target: "build test") + } + } + stage('Packetbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } + steps { + mageTarget(context: "Packetbeat OSS Mac OS X", directory: "packetbeat", target: "build unitTest") + } + post { + always { + delete() + } + } + } + stage('Packetbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin(context: "Packetbeat OSS Windows", directory: "packetbeat", target: "build unitTest") } } } @@ -579,7 +610,7 @@ pipeline { stages { stage('Dockerlogbeat'){ steps { - mageTarget(context: "Elastic Docker Logging Driver Plugin unit tests", directory: "x-pack/dockerlogbeat", target: "update build test") + mageTarget(context: "Elastic Docker Logging Driver Plugin unit tests", directory: "x-pack/dockerlogbeat", target: "build test") } } } @@ -641,7 +672,7 @@ pipeline { steps { mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "update build test") withEnv(["GO_VERSION=1.13.1"]){ - makeTarget(context: "Functionbeat x-pack Linux", directory: 'x-pack/functionbeat', target: "test-gcp-functions") + mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "testGCPFunctions") } } } @@ -690,7 +721,7 @@ pipeline { stages { stage('Journalbeat oss'){ steps { - makeTarget(context: "Journalbeat Linux", directory: 'journalbeat', target: "testsuite") + mageTarget(context: "Journalbeat Linux", directory: "journalbeat", target: "build goUnitTest") } } } diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index 115547619406..73110b173546 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -31,16 +31,18 @@ import ( // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/common" // mage:import - _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" - // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import + "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" + // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" ) func init() { common.RegisterCheckDeps(Update) unittest.RegisterGoTestDeps(fieldsYML) + integtest.RegisterGoTestDeps(fieldsYML) + integtest.RegisterPythonTestDeps(Dashboards) devtools.BeatDescription = "Audit the activities of users and processes on your system." } diff --git a/dev-tools/mage/integtest.go b/dev-tools/mage/integtest.go index 396c010dbc20..16321fa90326 100644 --- a/dev-tools/mage/integtest.go +++ b/dev-tools/mage/integtest.go @@ -213,7 +213,7 @@ func NewIntegrationRunners(path string, passInEnv map[string]string) (Integratio return runners, nil } -// NewDockerIntegrationRunner returns an intergration runner configured only for docker. +// NewDockerIntegrationRunner returns an integration runner configured only for docker. func NewDockerIntegrationRunner(passThroughEnvVars ...string) (*IntegrationRunner, error) { cwd, err := os.Getwd() if err != nil { diff --git a/dev-tools/mage/integtest_docker.go b/dev-tools/mage/integtest_docker.go index afc05a16dc1a..da7ecf33c375 100644 --- a/dev-tools/mage/integtest_docker.go +++ b/dev-tools/mage/integtest_docker.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "os" + "path" "path/filepath" "runtime" "strings" @@ -156,6 +157,13 @@ func (d *DockerIntegrationTester) Test(_ string, mageTarget string, env map[stri func (d *DockerIntegrationTester) InsideTest(test func() error) error { // Fix file permissions after test is done writing files as root. if runtime.GOOS != "windows" { + repo, err := GetProjectRepoInfo() + if err != nil { + return err + } + + // Handle virtualenv and the current project dir. + defer DockerChown(path.Join(repo.RootDir, "build")) defer DockerChown(".") } return test() diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index 9c15e1f7e5f4..c8a12fdfab62 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -132,9 +132,7 @@ func PythonNoseTest(params PythonTestArgs) error { nosetestsOptions := []string{ "--process-timeout=90", "--with-timer", - } - if mg.Verbose() { - nosetestsOptions = append(nosetestsOptions, "-v") + "-v", } if params.XUnitReportFile != "" { nosetestsOptions = append(nosetestsOptions, @@ -188,7 +186,7 @@ func PythonNoseTestForModule(params PythonTestArgs) error { // PythonVirtualenv constructs a virtualenv that contains the given modules as // defined in the requirements file pointed to by requirementsTxt. It returns -// the path to the virutalenv. +// the path to the virtualenv. func PythonVirtualenv() (string, error) { pythonVirtualenvLock.Lock() defer pythonVirtualenvLock.Unlock() @@ -221,20 +219,28 @@ func PythonVirtualenv() (string, error) { } pip := virtualenvPath(ve, "pip") - args := []string{"install"} - if !mg.Verbose() { - args = append(args, "--quiet") + pipUpgrade := func(pkg string) error { + return sh.RunWith(env, pip, "install", "-U", pkg) } - for _, req := range reqs { - args = append(args, "-Ur", req) + + // Ensure we are using the latest pip version. + if err = pipUpgrade("pip"); err != nil { + fmt.Printf("warn: failed to upgrade pip (ignoring): %v", err) } // First ensure that wheel is installed so that bdists build cleanly. - if err = sh.RunWith(env, pip, "install", "-U", "wheel"); err != nil { + if err = pipUpgrade("wheel"); err != nil { return "", err } // Execute pip to install the dependencies. + args := []string{"install"} + if !mg.Verbose() { + args = append(args, "--quiet") + } + for _, req := range reqs { + args = append(args, "-Ur", req) + } if err := sh.RunWith(env, pip, args...); err != nil { return "", err } diff --git a/filebeat/tests/system/test_input.py b/filebeat/tests/system/test_input.py index 8aebe49f4d0a..7192fd956d54 100644 --- a/filebeat/tests/system/test_input.py +++ b/filebeat/tests/system/test_input.py @@ -713,6 +713,7 @@ def test_path_based_identity_tracking(self): # assert that renaming of the file went undetected assert not self.log_contains("File rename was detected:" + testfile + " -> " + renamedfile) + @unittest.skip("Skipped as flaky: https://github.com/elastic/beats/issues/20010") @unittest.skipIf(sys.platform.startswith("win"), "inode_marker is not supported on windows") def test_inode_marker_based_identity_tracking(self): """ diff --git a/heartbeat/magefile.go b/heartbeat/magefile.go index 298ee0443d1a..6d2bbf8c2f34 100644 --- a/heartbeat/magefile.go +++ b/heartbeat/magefile.go @@ -33,10 +33,10 @@ import ( // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/common" // mage:import - "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" - // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import + "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" + // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" ) diff --git a/libbeat/tests/system/test_http.py b/libbeat/tests/system/test_http.py index c2379676da59..5c1baa81bd22 100644 --- a/libbeat/tests/system/test_http.py +++ b/libbeat/tests/system/test_http.py @@ -24,7 +24,7 @@ def test_root(self): r = requests.get("http://localhost:5066") assert r.status_code == 200 - data = json.loads(r.content) + data = json.loads(r.content.decode('utf_8')) assert data["beat"] == "mockbeat" assert data["version"] == "9.9.9" @@ -36,7 +36,7 @@ def test_stats(self): r = requests.get("http://localhost:5066/stats") assert r.status_code == 200 - data = json.loads(r.content) + data = json.loads(r.content.decode('utf_8')) # Test one data point assert data["libbeat"]["config"]["scans"] == 0 diff --git a/magefile.go b/magefile.go index ac4614e2cdee..75c6d60faef6 100644 --- a/magefile.go +++ b/magefile.go @@ -135,7 +135,6 @@ func CheckLicenseHeaders() error { licenser.Exclude("x-pack"), licenser.Exclude("generator/_templates/beat/{beat}"), licenser.Exclude("generator/_templates/metricbeat/{beat}"), - licenser.Exclude("generator/_templates/beat/{beat}"), ), licenser( licenser.Check(), diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go index 00622696bc48..fd00c932ab46 100644 --- a/packetbeat/magefile.go +++ b/packetbeat/magefile.go @@ -35,7 +35,7 @@ import ( // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/common" // mage:import - _ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" + "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest/notests" // mage:import @@ -44,6 +44,7 @@ import ( func init() { common.RegisterCheckDeps(Update) + unittest.RegisterPythonTestDeps(fieldsYML, Dashboards) devtools.BeatDescription = "Packetbeat analyzes network traffic and sends the data to Elasticsearch." } diff --git a/packetbeat/tests/system/packetbeat.py b/packetbeat/tests/system/packetbeat.py index 24b59a32b9cc..0a05e49287b6 100644 --- a/packetbeat/tests/system/packetbeat.py +++ b/packetbeat/tests/system/packetbeat.py @@ -126,7 +126,7 @@ def read_output(self, types=None, required_fields=None): jsons = [] - with open(os.path.join(self.working_dir, output_file), "r") as f: + with open(os.path.join(self.working_dir, output_file), "r", encoding='utf_8') as f: for line in f: document = self.flatten_object(json.loads(line), self.dict_fields) if not types or document["type"] in types: diff --git a/x-pack/libbeat/magefile.go b/x-pack/libbeat/magefile.go index 01e9b4d7f2d4..2b8d8f35d8ee 100644 --- a/x-pack/libbeat/magefile.go +++ b/x-pack/libbeat/magefile.go @@ -12,10 +12,10 @@ import ( // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/common" // mage:import - _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" - // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" + // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" )