Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CI] Optimise stash/unstash performance #18473

Merged
merged 14 commits into from
May 14, 2020
Merged

[CI] Optimise stash/unstash performance #18473

Changes from 12 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
fbd8fb8
[CI] Optimise stash/unstash performance
v1v May 13, 2020
86ac4c5
[CI] Use google storage
v1v May 13, 2020
0ec32a7
ci: flatten files
v1v May 13, 2020
1b612aa
ci: fix the permission denied after the unzip
v1v May 13, 2020
04c2db7
ci: let's be explicit
v1v May 13, 2020
44a5679
ci: zip does not compress the .git folder
v1v May 13, 2020
d4b1421
ci: fix the permission denied after the unzip
v1v May 13, 2020
3563a8d
ci: use tar to support restore permissions
v1v May 13, 2020
df947c2
ci: fix the file changed as we read it
v1v May 13, 2020
8dfa353
Revert "ci: fix the file changed as we read it"
v1v May 13, 2020
ba137ae
ci: fix the file changed as we read it
v1v May 13, 2020
4a8ceec
Tweak some of the stash/unstash arguments
v1v May 14, 2020
aabc3de
ci: refactored with the shared library steps
v1v May 14, 2020
0104e63
Update Jenkinsfile
v1v May 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 48 additions & 5 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ pipeline {
DOCKER_REGISTRY = 'docker.elastic.co'
AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
RUNBLD_DISABLE_NOTIFICATIONS = 'true'
JOB_GCS_BUCKET = 'beats-ci-temp'
v1v marked this conversation as resolved.
Show resolved Hide resolved
JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
}
options {
timeout(time: 2, unit: 'HOURS')
Expand Down Expand Up @@ -57,7 +59,7 @@ pipeline {
pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ])
deleteDir()
gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true)
stash allowEmpty: true, name: 'source', useDefaultExcludes: false
stashV2('source')
dir("${BASE_DIR}"){
loadConfigEnvVars()
}
Expand Down Expand Up @@ -722,7 +724,7 @@ def withBeatsEnv(boolean archive, Closure body) {
"DOCKER_PULL=0",
]) {
deleteDir()
unstash 'source'
unstashV2('source')
if(isDockerInstalled()){
dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
}
Expand Down Expand Up @@ -763,7 +765,7 @@ def withBeatsEnvWin(Closure body) {
"RACE_DETECTOR=true",
]){
deleteDir()
unstash 'source'
unstashV2('source')
dir("${env.BASE_DIR}"){
installTools()
try {
Expand Down Expand Up @@ -1027,7 +1029,7 @@ def terraformCleanup(String stashName, String directory) {
stage("Remove cloud scenarios in ${directory}"){
withCloudTestEnv() {
withBeatsEnv(false) {
unstash "terraform-${stashName}"
unstash("terraform-${stashName}")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cosmetic change

retry(2) {
sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}")
}
Expand Down Expand Up @@ -1176,7 +1178,7 @@ def runbld() {
// Unstash the test reports
stashedTestReports.each { k, v ->
dir(k) {
unstash v
unstash(v)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cosmetic change

}
}
sh(label: 'Process JUnit reports with runbld',
Expand All @@ -1190,3 +1192,44 @@ def runbld() {
}
}
}

def stashV2(String name) {
def filename = "${name}.zip"
writeFile file: "${filename}", text: ''
def command = "tar --exclude=${filename} -czf ${filename} ."
if(isUnix()) {
sh(label: 'Archive', script: command)
} else {
bat(label: 'Archive', script: command)
}
googleStorageUpload(
bucket: "gs://${JOB_GCS_BUCKET}/${JOB_NAME}-${BUILD_NUMBER}/${name}",
credentialsId: "${JOB_GCS_CREDENTIALS}",
pattern: "${filename}",
sharedPublicly: false,
showInline: true
)
v1v marked this conversation as resolved.
Show resolved Hide resolved
if(isUnix()) {
sh(label: 'Delete zip', script: "rm ${filename}", returnStatus: true)
} else {
bat(label: 'Delete zip', script: "del ${filename}", returnStatus: true)
}
}

def unstashV2(String name) {
def filename = "${name}.zip"
def command = "tar -xpf ${filename}"
googleStorageDownload(
bucketUri: "gs://${JOB_GCS_BUCKET}/${JOB_NAME}-${BUILD_NUMBER}/${name}/${filename}",
credentialsId: "${JOB_GCS_CREDENTIALS}",
v1v marked this conversation as resolved.
Show resolved Hide resolved
localDirectory: '',
pathPrefix: "${JOB_NAME}-${BUILD_NUMBER}/${name}/"
)
if(isUnix()) {
command += "; rm ${filename}"
sh(label: 'Extract', script: command)
} else {
bat(label: 'Extract', script: command)
bat(label: 'Delete zip', script: "del ${filename}", returnStatus: true)
}
}