Skip to content
This repository has been archived by the owner on Mar 31, 2024. It is now read-only.

Commit

Permalink
[jenkins] refer to sizes in most pipeline code (elastic#62082)
Browse files Browse the repository at this point in the history
* [jenkins] refer to sizes in most pipeline code

* switch back to `linux && immutable` for small instances

Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
  • Loading branch information
3 people committed Apr 4, 2020
1 parent 3ca04c0 commit 627cffc
Show file tree
Hide file tree
Showing 5 changed files with 30 additions and 15 deletions.
2 changes: 1 addition & 1 deletion .ci/Jenkinsfile_coverage
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ kibanaPipeline(timeoutMinutes: 180) {
'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
]),
])
workers.base(name: 'coverage-worker', label: 'tests-l', ramDisk: false, bootstrapped: false) {
workers.base(name: 'coverage-worker', size: 'l', ramDisk: false, bootstrapped: false) {
kibanaPipeline.downloadCoverageArtifacts()
kibanaPipeline.bash(
'''
Expand Down
4 changes: 2 additions & 2 deletions .ci/Jenkinsfile_visual_baseline
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ kibanaPipeline(timeoutMinutes: 120) {
catchError {
parallel([
'oss-visualRegression': {
workers.ci(name: 'oss-visualRegression', label: 'linux && immutable', ramDisk: false) {
workers.ci(name: 'oss-visualRegression', size: 's', ramDisk: false) {
kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')(1)
}
},
'xpack-visualRegression': {
workers.ci(name: 'xpack-visualRegression', label: 'linux && immutable', ramDisk: false) {
workers.ci(name: 'xpack-visualRegression', size: 's', ramDisk: false) {
kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')(1)
}
},
Expand Down
2 changes: 1 addition & 1 deletion .ci/es-snapshots/Jenkinsfile_build_es
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION
timeout(time: 120, unit: 'MINUTES') {
timestamps {
ansiColor('xterm') {
node('linux && immutable') {
node(workers.label('s')) {
catchErrors {
def VERSION
def SNAPSHOT_ID
Expand Down
2 changes: 1 addition & 1 deletion .ci/es-snapshots/Jenkinsfile_verify_es
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ kibanaPipeline(timeoutMinutes: 120) {
}

def promoteSnapshot(snapshotVersion, snapshotId) {
node('linux && immutable') {
node(workers.label('s')) {
esSnapshots.promote(snapshotVersion, snapshotId)
}
}
35 changes: 25 additions & 10 deletions vars/workers.groovy
Original file line number Diff line number Diff line change
@@ -1,23 +1,38 @@
// "Workers" in this file will spin up an instance, do some setup etc depending on the configuration, and then execute some work that you define
// e.g. workers.base(name: 'my-worker') { sh "echo 'ready to execute some kibana scripts'" }

def label(size) {
switch(size) {
case 's':
return 'linux && immutable'
case 'l':
return 'tests-l'
case 'xl':
return 'tests-xl'
case 'xxl':
return 'tests-xxl'
}

error "unknown size '${size}'"
}

/*
The base worker that all of the others use. Will clone the scm (assumed to be kibana), and run kibana bootstrap processes by default.
Parameters:
label - gobld/agent label to use, e.g. 'linux && immutable'
size - size of worker label to use, e.g. 's' or 'xl'
ramDisk - Should the workspace be mounted in memory? Default: true
bootstrapped - If true, download kibana dependencies, run kbn bootstrap, etc. Default: true
name - Name of the worker for display purposes, filenames, etc.
scm - Jenkins scm configuration for checking out code. Use `null` to disable checkout. Default: inherited from job
*/
def base(Map params, Closure closure) {
def config = [label: '', ramDisk: true, bootstrapped: true, name: 'unnamed-worker', scm: scm] + params
if (!config.label) {
error "You must specify an agent label, such as 'tests-xl' or 'linux && immutable', when using workers.base()"
def config = [size: '', ramDisk: true, bootstrapped: true, name: 'unnamed-worker', scm: scm] + params
if (!config.size) {
error "You must specify an agent size, such as 'xl' or 's', when using workers.base()"
}

node(config.label) {
node(label(config.size)) {
agentInfo.print()

if (config.ramDisk) {
Expand Down Expand Up @@ -88,7 +103,7 @@ def ci(Map params, Closure closure) {
// Worker for running the current intake jobs. Just runs a single script after bootstrap.
def intake(jobName, String script) {
return {
ci(name: jobName, label: 'linux && immutable', ramDisk: false) {
ci(name: jobName, size: 's', ramDisk: false) {
withEnv(["JOB=${jobName}"]) {
runbld(script, "Execute ${jobName}")
}
Expand All @@ -99,7 +114,7 @@ def intake(jobName, String script) {
// Worker for running functional tests. Runs a setup process (e.g. the kibana build) then executes a map of closures in parallel (e.g. one for each ciGroup)
def functional(name, Closure setup, Map processes) {
return {
parallelProcesses(name: name, setup: setup, processes: processes, delayBetweenProcesses: 20, label: 'tests-xl')
parallelProcesses(name: name, setup: setup, processes: processes, delayBetweenProcesses: 20, size: 'xl')
}
}

Expand All @@ -111,12 +126,12 @@ def functional(name, Closure setup, Map processes) {
setup: Closure to execute after the agent is bootstrapped, before starting the parallel work
processes: Map of closures that will execute in parallel after setup. Each closure is passed a unique number.
delayBetweenProcesses: Number of seconds to wait between starting the parallel processes. Useful to spread the load of heavy init processes, e.g. Elasticsearch starting up. Default: 0
label: gobld/agent label to use, e.g. 'linux && immutable'. Default: 'tests-xl', a 32 CPU machine used for running many functional test suites in parallel
size: size of worker label to use, e.g. 's' or 'xl'
*/
def parallelProcesses(Map params) {
def config = [name: 'parallel-worker', setup: {}, processes: [:], delayBetweenProcesses: 0, label: 'tests-xl'] + params
def config = [name: 'parallel-worker', setup: {}, processes: [:], delayBetweenProcesses: 0, size: 'xl'] + params

ci(label: config.label, name: config.name) {
ci(size: config.size, name: config.name) {
config.setup()

def nextProcessNumber = 1
Expand Down

0 comments on commit 627cffc

Please sign in to comment.