From 4af63650a975f9ad23bc8bb6fae44b295f6e2021 Mon Sep 17 00:00:00 2001 From: spalger Date: Tue, 31 Mar 2020 14:10:34 -0700 Subject: [PATCH 1/2] [jenkins] refer to sizes in most pipeline code --- .ci/Jenkinsfile_coverage | 2 +- .ci/Jenkinsfile_visual_baseline | 4 +-- .ci/es-snapshots/Jenkinsfile_build_es | 2 +- .ci/es-snapshots/Jenkinsfile_verify_es | 2 +- vars/workers.groovy | 35 ++++++++++++++++++-------- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.ci/Jenkinsfile_coverage b/.ci/Jenkinsfile_coverage index 6b8dc31bab34e..f2a58e7b6a7ac 100644 --- a/.ci/Jenkinsfile_coverage +++ b/.ci/Jenkinsfile_coverage @@ -44,7 +44,7 @@ kibanaPipeline(timeoutMinutes: 180) { 'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10), ]), ]) - workers.base(name: 'coverage-worker', label: 'tests-l', ramDisk: false, bootstrapped: false) { + workers.base(name: 'coverage-worker', size: 'l', ramDisk: false, bootstrapped: false) { kibanaPipeline.downloadCoverageArtifacts() kibanaPipeline.bash( ''' diff --git a/.ci/Jenkinsfile_visual_baseline b/.ci/Jenkinsfile_visual_baseline index 5c13ccccd9c6f..815c1345bbb68 100644 --- a/.ci/Jenkinsfile_visual_baseline +++ b/.ci/Jenkinsfile_visual_baseline @@ -7,12 +7,12 @@ kibanaPipeline(timeoutMinutes: 120) { catchError { parallel([ 'oss-visualRegression': { - workers.ci(name: 'oss-visualRegression', label: 'linux && immutable', ramDisk: false) { + workers.ci(name: 'oss-visualRegression', size: 's', ramDisk: false) { kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')(1) } }, 'xpack-visualRegression': { - workers.ci(name: 'xpack-visualRegression', label: 'linux && immutable', ramDisk: false) { + workers.ci(name: 'xpack-visualRegression', size: 's', ramDisk: false) { kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')(1) } }, diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index a00bcb3bbc946..a3470cd750738 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -25,7 +25,7 @@ def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION timeout(time: 120, unit: 'MINUTES') { timestamps { ansiColor('xterm') { - node('linux && immutable') { + node(workers.label('s')) { catchErrors { def VERSION def SNAPSHOT_ID diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es b/.ci/es-snapshots/Jenkinsfile_verify_es index ce472a404c053..ade79f27e10e9 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es +++ b/.ci/es-snapshots/Jenkinsfile_verify_es @@ -61,7 +61,7 @@ kibanaPipeline(timeoutMinutes: 120) { } def promoteSnapshot(snapshotVersion, snapshotId) { - node('linux && immutable') { + node(workers.label('s')) { esSnapshots.promote(snapshotVersion, snapshotId) } } diff --git a/vars/workers.groovy b/vars/workers.groovy index c5638f2624fe5..0f8d50e487281 100644 --- a/vars/workers.groovy +++ b/vars/workers.groovy @@ -1,23 +1,38 @@ // "Workers" in this file will spin up an instance, do some setup etc depending on the configuration, and then execute some work that you define // e.g. workers.base(name: 'my-worker') { sh "echo 'ready to execute some kibana scripts'" } +def label(size) { + switch(size) { + case 's': + return 'ubuntu-18.04 && immutable' + case 'l': + return 'tests-l' + case 'xl': + return 'tests-xl' + case 'xxl': + return 'tests-xxl' + } + + error "unknown size '${size}'" +} + /* The base worker that all of the others use. Will clone the scm (assumed to be kibana), and run kibana bootstrap processes by default. Parameters: - label - gobld/agent label to use, e.g. 'linux && immutable' + size - size of worker label to use, e.g. 's' or 'xl' ramDisk - Should the workspace be mounted in memory? Default: true bootstrapped - If true, download kibana dependencies, run kbn bootstrap, etc. Default: true name - Name of the worker for display purposes, filenames, etc. scm - Jenkins scm configuration for checking out code. Use `null` to disable checkout. Default: inherited from job */ def base(Map params, Closure closure) { - def config = [label: '', ramDisk: true, bootstrapped: true, name: 'unnamed-worker', scm: scm] + params - if (!config.label) { - error "You must specify an agent label, such as 'tests-xl' or 'linux && immutable', when using workers.base()" + def config = [size: '', ramDisk: true, bootstrapped: true, name: 'unnamed-worker', scm: scm] + params + if (!config.size) { + error "You must specify an agent size, such as 'xl' or 's', when using workers.base()" } - node(config.label) { + node(label(config.size)) { agentInfo.print() if (config.ramDisk) { @@ -88,7 +103,7 @@ def ci(Map params, Closure closure) { // Worker for running the current intake jobs. Just runs a single script after bootstrap. def intake(jobName, String script) { return { - ci(name: jobName, label: 'linux && immutable', ramDisk: false) { + ci(name: jobName, size: 's', ramDisk: false) { withEnv(["JOB=${jobName}"]) { runbld(script, "Execute ${jobName}") } @@ -99,7 +114,7 @@ def intake(jobName, String script) { // Worker for running functional tests. Runs a setup process (e.g. the kibana build) then executes a map of closures in parallel (e.g. one for each ciGroup) def functional(name, Closure setup, Map processes) { return { - parallelProcesses(name: name, setup: setup, processes: processes, delayBetweenProcesses: 20, label: 'tests-xl') + parallelProcesses(name: name, setup: setup, processes: processes, delayBetweenProcesses: 20, size: 'xl') } } @@ -111,12 +126,12 @@ def functional(name, Closure setup, Map processes) { setup: Closure to execute after the agent is bootstrapped, before starting the parallel work processes: Map of closures that will execute in parallel after setup. Each closure is passed a unique number. delayBetweenProcesses: Number of seconds to wait between starting the parallel processes. Useful to spread the load of heavy init processes, e.g. Elasticsearch starting up. Default: 0 - label: gobld/agent label to use, e.g. 'linux && immutable'. Default: 'tests-xl', a 32 CPU machine used for running many functional test suites in parallel + size: size of worker label to use, e.g. 's' or 'xl' */ def parallelProcesses(Map params) { - def config = [name: 'parallel-worker', setup: {}, processes: [:], delayBetweenProcesses: 0, label: 'tests-xl'] + params + def config = [name: 'parallel-worker', setup: {}, processes: [:], delayBetweenProcesses: 0, size: 'xl'] + params - ci(label: config.label, name: config.name) { + ci(size: config.size, name: config.name) { config.setup() def nextProcessNumber = 1 From 5a00bddf316096fde22ca1c2ab579b3134225727 Mon Sep 17 00:00:00 2001 From: spalger Date: Tue, 31 Mar 2020 14:56:53 -0700 Subject: [PATCH 2/2] switch back to `linux && immutable` for small instances --- vars/workers.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/workers.groovy b/vars/workers.groovy index 0f8d50e487281..1c55c676d9425 100644 --- a/vars/workers.groovy +++ b/vars/workers.groovy @@ -4,7 +4,7 @@ def label(size) { switch(size) { case 's': - return 'ubuntu-18.04 && immutable' + return 'linux && immutable' case 'l': return 'tests-l' case 'xl':