Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[jenkins] refer to sizes in most pipeline code #62082

Merged
merged 3 commits into from
Apr 4, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ci/Jenkinsfile_coverage
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ kibanaPipeline(timeoutMinutes: 180) {
'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
]),
])
workers.base(name: 'coverage-worker', label: 'tests-l', ramDisk: false, bootstrapped: false) {
workers.base(name: 'coverage-worker', size: 'l', ramDisk: false, bootstrapped: false) {
kibanaPipeline.downloadCoverageArtifacts()
kibanaPipeline.bash(
'''
Expand Down
4 changes: 2 additions & 2 deletions .ci/Jenkinsfile_visual_baseline
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ kibanaPipeline(timeoutMinutes: 120) {
catchError {
parallel([
'oss-visualRegression': {
workers.ci(name: 'oss-visualRegression', label: 'linux && immutable', ramDisk: false) {
workers.ci(name: 'oss-visualRegression', size: 's', ramDisk: false) {
kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')(1)
}
},
'xpack-visualRegression': {
workers.ci(name: 'xpack-visualRegression', label: 'linux && immutable', ramDisk: false) {
workers.ci(name: 'xpack-visualRegression', size: 's', ramDisk: false) {
kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')(1)
}
},
Expand Down
2 changes: 1 addition & 1 deletion .ci/es-snapshots/Jenkinsfile_build_es
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION
timeout(time: 120, unit: 'MINUTES') {
timestamps {
ansiColor('xterm') {
node('linux && immutable') {
node(workers.label('s')) {
catchErrors {
def VERSION
def SNAPSHOT_ID
Expand Down
2 changes: 1 addition & 1 deletion .ci/es-snapshots/Jenkinsfile_verify_es
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ kibanaPipeline(timeoutMinutes: 120) {
}

def promoteSnapshot(snapshotVersion, snapshotId) {
node('linux && immutable') {
node(workers.label('s')) {
esSnapshots.promote(snapshotVersion, snapshotId)
}
}
35 changes: 25 additions & 10 deletions vars/workers.groovy
Original file line number Diff line number Diff line change
@@ -1,23 +1,38 @@
// "Workers" in this file will spin up an instance, do some setup etc depending on the configuration, and then execute some work that you define
// e.g. workers.base(name: 'my-worker') { sh "echo 'ready to execute some kibana scripts'" }

def label(size) {
switch(size) {
case 's':
return 'linux && immutable'
case 'l':
return 'tests-l'
case 'xl':
return 'tests-xl'
case 'xxl':
return 'tests-xxl'
}

error "unknown size '${size}'"
}

/*
The base worker that all of the others use. Will clone the scm (assumed to be kibana), and run kibana bootstrap processes by default.

Parameters:
label - gobld/agent label to use, e.g. 'linux && immutable'
size - size of worker label to use, e.g. 's' or 'xl'
ramDisk - Should the workspace be mounted in memory? Default: true
bootstrapped - If true, download kibana dependencies, run kbn bootstrap, etc. Default: true
name - Name of the worker for display purposes, filenames, etc.
scm - Jenkins scm configuration for checking out code. Use `null` to disable checkout. Default: inherited from job
*/
def base(Map params, Closure closure) {
def config = [label: '', ramDisk: true, bootstrapped: true, name: 'unnamed-worker', scm: scm] + params
if (!config.label) {
error "You must specify an agent label, such as 'tests-xl' or 'linux && immutable', when using workers.base()"
def config = [size: '', ramDisk: true, bootstrapped: true, name: 'unnamed-worker', scm: scm] + params
if (!config.size) {
error "You must specify an agent size, such as 'xl' or 's', when using workers.base()"
}

node(config.label) {
node(label(config.size)) {
agentInfo.print()

if (config.ramDisk) {
Expand Down Expand Up @@ -88,7 +103,7 @@ def ci(Map params, Closure closure) {
// Worker for running the current intake jobs. Just runs a single script after bootstrap.
def intake(jobName, String script) {
return {
ci(name: jobName, label: 'linux && immutable', ramDisk: false) {
ci(name: jobName, size: 's', ramDisk: false) {
withEnv(["JOB=${jobName}"]) {
runbld(script, "Execute ${jobName}")
}
Expand All @@ -99,7 +114,7 @@ def intake(jobName, String script) {
// Worker for running functional tests. Runs a setup process (e.g. the kibana build) then executes a map of closures in parallel (e.g. one for each ciGroup)
def functional(name, Closure setup, Map processes) {
return {
parallelProcesses(name: name, setup: setup, processes: processes, delayBetweenProcesses: 20, label: 'tests-xl')
parallelProcesses(name: name, setup: setup, processes: processes, delayBetweenProcesses: 20, size: 'xl')
}
}

Expand All @@ -111,12 +126,12 @@ def functional(name, Closure setup, Map processes) {
setup: Closure to execute after the agent is bootstrapped, before starting the parallel work
processes: Map of closures that will execute in parallel after setup. Each closure is passed a unique number.
delayBetweenProcesses: Number of seconds to wait between starting the parallel processes. Useful to spread the load of heavy init processes, e.g. Elasticsearch starting up. Default: 0
label: gobld/agent label to use, e.g. 'linux && immutable'. Default: 'tests-xl', a 32 CPU machine used for running many functional test suites in parallel
size: size of worker label to use, e.g. 's' or 'xl'
*/
def parallelProcesses(Map params) {
def config = [name: 'parallel-worker', setup: {}, processes: [:], delayBetweenProcesses: 0, label: 'tests-xl'] + params
def config = [name: 'parallel-worker', setup: {}, processes: [:], delayBetweenProcesses: 0, size: 'xl'] + params

ci(label: config.label, name: config.name) {
ci(size: config.size, name: config.name) {
config.setup()

def nextProcessNumber = 1
Expand Down