diff --git a/.ci/beats-tester.groovy b/.ci/beats-tester.groovy index 91781a98d31..3650246e966 100644 --- a/.ci/beats-tester.groovy +++ b/.ci/beats-tester.groovy @@ -54,7 +54,6 @@ pipeline { options { skipDefaultCheckout() } when { branch 'master' } steps { - // TODO: to use the git commit that triggered the upstream build runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT") } } @@ -62,7 +61,6 @@ pipeline { options { skipDefaultCheckout() } when { branch '*.x' } steps { - // TODO: to use the git commit that triggered the upstream build runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT") } } @@ -79,7 +77,7 @@ pipeline { options { skipDefaultCheckout() } when { not { - allOf { + anyOf { branch comparator: 'REGEXP', pattern: '(master|.*x)' changeRequest() } @@ -96,14 +94,28 @@ pipeline { } def runBeatsTesterJob(Map args = [:]) { - if (args.apm && args.beats) { + def apm = args.get('apm', '') + def beats = args.get('beats', '') + def version = args.version + + if (isUpstreamTrigger()) { + copyArtifacts(filter: 'beats-tester.properties', + flatten: true, + projectName: "Beats/packaging/${env.JOB_BASE_NAME}", + selector: upstream(fallbackToLastSuccessful: true)) + def props = readProperties(file: 'beats-tester.properties') + apm = props.get('APM_URL_BASE', '') + beats = props.get('BEATS_URL_BASE', '') + version = props.get('VERSION', '8.0.0-SNAPSHOT') + } + if (apm?.trim() || beats?.trim()) { build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ - string(name: 'APM_URL_BASE', value: args.apm), - string(name: 'BEATS_URL_BASE', value: args.beats), - string(name: 'VERSION', value: args.version) + string(name: 'APM_URL_BASE', value: apm), + string(name: 'BEATS_URL_BASE', value: beats), + string(name: 'VERSION', value: version) ]) } else { - build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: args.version) ]) + build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: version) ]) } -} \ No newline at end of file +} diff --git a/.ci/jobs/beats-tester.yml b/.ci/jobs/beats-tester.yml index 808123a225e..14cc1007c56 100644 --- a/.ci/jobs/beats-tester.yml +++ b/.ci/jobs/beats-tester.yml @@ -1,8 +1,8 @@ --- - job: name: Beats/beats-tester - display-name: Beats Tester - description: Run the beats-tester + display-name: Beats Tester orchestrator + description: Orchestrate the beats-tester when packaging finished successfully view: Beats disabled: false project-type: multibranch diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml index 27095b2fecb..54952b06e7c 100644 --- a/.ci/jobs/beats.yml +++ b/.ci/jobs/beats.yml @@ -35,6 +35,9 @@ - exact-name: name: 'master' case-sensitive: true + - regex-name: + regex: '6\.[x89]' + case-sensitive: true - regex-name: regex: '7\.[x789]' case-sensitive: true diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index e86906fd8bb..02adc1a06b8 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -65,8 +65,29 @@ pipeline { options { skipDefaultCheckout() } steps { deleteDir() - gitCheckout(basedir: "${BASE_DIR}") + script { + if(isUpstreamTrigger()) { + try { + copyArtifacts(filter: 'packaging.properties', + flatten: true, + projectName: "Beats/beats/${env.JOB_BASE_NAME}", + selector: upstream(fallbackToLastSuccessful: true)) + def props = readProperties(file: 'packaging.properties') + gitCheckout(basedir: "${BASE_DIR}", branch: props.COMMIT) + } catch(err) { + // Fallback to the head of the branch as used to be. + gitCheckout(basedir: "${BASE_DIR}") + } + } else { + gitCheckout(basedir: "${BASE_DIR}") + } + } setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim()) + withMageEnv(){ + dir("${BASE_DIR}"){ + setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) + } + } stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") } } @@ -167,6 +188,17 @@ pipeline { } } } + post { + success { + writeFile(file: 'beats-tester.properties', + text: """\ + ## To be consumed by the beats-tester pipeline + COMMIT=${env.GIT_BASE_COMMIT} + BEATS_URL_BASE=https://storage.googleapis.com/${env.JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT} + VERSION=${env.BEAT_VERSION}-SNAPSHOT""".stripIndent()) // stripIdent() requires '''/ + archiveArtifacts artifacts: 'beats-tester.properties' + } + } } } } @@ -192,7 +224,7 @@ def pushCIDockerImages(){ } def tagAndPush(beatName){ - def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim() + def libbetaVer = env.BEAT_VERSION def aliasVersion = "" if("${env.SNAPSHOT}" == "true"){ aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version @@ -346,7 +378,7 @@ def publishPackages(baseDir){ uploadPackages("${bucketUri}/${beatsFolderName}", baseDir) // Copy those files to another location with the sha commit to test them - // aftewords. + // afterward. bucketUri = "gs://${JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}" uploadPackages("${bucketUri}/${beatsFolderName}", baseDir) } diff --git a/Jenkinsfile b/Jenkinsfile index 6fff22bfa09..82bd3e0cef2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,6 +77,7 @@ pipeline { withGithubNotify(context: "Lint") { withBeatsEnv(archive: false, id: "lint") { dumpVariables() + setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) cmd(label: "make check-python", script: "make check-python") cmd(label: "make check-go", script: "make check-go") cmd(label: "Check for changes", script: "make check-no-changes") @@ -123,6 +124,12 @@ pipeline { } } post { + success { + writeFile(file: 'packaging.properties', text: """## To be consumed by the packaging pipeline +COMMIT=${env.GIT_BASE_COMMIT} +VERSION=${env.VERSION}-SNAPSHOT""") + archiveArtifacts artifacts: 'packaging.properties' + } always { deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") @@ -344,6 +351,7 @@ def withBeatsEnv(Map args = [:], Closure body) { } catch(err) { // Upload the generated files ONLY if the step failed. This will avoid any overhead with Google Storage upload = true + error("Error '${err.toString()}'") } finally { if (archive) { archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) diff --git a/Makefile b/Makefile index b00cded4f9f..bd0da94f129 100644 --- a/Makefile +++ b/Makefile @@ -191,6 +191,11 @@ python-env: test-apm: sh ./script/test_apm.sh +## get-version : Get the libbeat version +.PHONY: get-version +get-version: + @mage dumpVariables | grep 'beat_version' | cut -d"=" -f 2 | tr -d " " + ### Packaging targets #### ## snapshot : Builds a snapshot release. diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 48df00ee123..aa9006d7f68 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -86,7 +86,18 @@ stages: - "windows-10" branches: true ## for all the branches tags: true ## for all the tags - # windows-7: See https://github.com/elastic/beats/issues/19831 + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test auditbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags + #windows-7: See https://github.com/elastic/beats/issues/19831 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7" diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index 11f857f689c..e5b6397e364 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -74,7 +74,18 @@ stages: - "windows-10" branches: true ## for all the branches tags: true ## for all the tags - # windows-7: See https://github.com/elastic/beats/issues/22317 + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test filebeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags + #windows-7: See https://github.com/elastic/beats/issues/22317 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7" diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 1369801b205..5b1fd667319 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -94,6 +94,17 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tag + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test heartbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tag windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 0a9edd3cf0e..b90c187853e 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -79,6 +79,17 @@ stages: - "windows-10" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test metricbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index 64caca2fa41..687a4dde7c5 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -94,6 +94,17 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test packetbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml index 54c76a016d3..f237c26173d 100644 --- a/winlogbeat/Jenkinsfile.yml +++ b/winlogbeat/Jenkinsfile.yml @@ -47,7 +47,7 @@ stages: - "windows-2012" branches: true ## for all the branches tags: true ## for all the tags - # windows-10: See https://github.com/elastic/beats/issues/22046 + #windows-10: See https://github.com/elastic/beats/issues/22046 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-10" @@ -58,7 +58,7 @@ stages: # - "windows-10" # branches: true ## for all the branches # tags: true ## for all the tags - # windows-7: See https://github.com/elastic/beats/issues/22302 + #windows-7: See https://github.com/elastic/beats/issues/22302 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7" @@ -69,3 +69,14 @@ stages: # - "windows-7" # branches: true ## for all the branches # tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test winlogbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index e6faa549aeb..1bb01d201d9 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -95,6 +95,17 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/auditbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index 94d2bd9e9d2..75ef188e587 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -94,6 +94,17 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/elastic-agent for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index e84246a7f69..8d635787ba4 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -95,6 +95,17 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/filebeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index c90ed26ee6f..764c325fa4b 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -92,6 +92,17 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/functionbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 2e0bf8cf934..6144707c0ea 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -84,6 +84,17 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/metricbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/x-pack/metricbeat/module/googlecloud/billing/_meta/data.json b/x-pack/metricbeat/module/googlecloud/billing/_meta/data.json index 92ee77fb15b..5000711030a 100644 --- a/x-pack/metricbeat/module/googlecloud/billing/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/billing/_meta/data.json @@ -1,12 +1,8 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "cloud": { - "account": { - "id": "elastic-bi", - "name": "elastic-bi" - }, - "provider": "googlecloud" - }, + "cloud.account.id": "elastic-bi", + "cloud.account.name": "elastic-bi", + "cloud.provider": "gcp", "event": { "dataset": "googlecloud.billing", "duration": 115000, @@ -15,9 +11,9 @@ "googlecloud": { "billing": { "cost_type": "regular", - "invoice_month": "202008", + "invoice_month": "202010", "project_id": "elastic-fin-bi", - "total": 170.811692 + "total": 77.897328 } }, "metricset": { diff --git a/x-pack/metricbeat/module/googlecloud/billing/billing.go b/x-pack/metricbeat/module/googlecloud/billing/billing.go index a314df78100..16cdbccf090 100644 --- a/x-pack/metricbeat/module/googlecloud/billing/billing.go +++ b/x-pack/metricbeat/module/googlecloud/billing/billing.go @@ -278,7 +278,7 @@ func createEvents(rowItems []bigquery.Value, accountID string) mb.Event { } event.RootFields = common.MapStr{ - "cloud.provider": "googlecloud", + "cloud.provider": "gcp", "cloud.account.id": accountID, "cloud.account.name": accountID, } diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index d87247c96bc..8f913b291ec 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -95,6 +95,17 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/packetbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml index 5921a2cd915..d5d09c08873 100644 --- a/x-pack/winlogbeat/Jenkinsfile.yml +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -69,6 +69,17 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/winlogbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage.