diff --git a/Jenkinsfile b/Jenkinsfile index aa9507c17811..ec6793c80a8c 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-08-11T12:19:24.817346 +// Generated at 2022-08-11T13:17:04.679404 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -5518,78 +5518,239 @@ def deploy_docs() { def deploy() { stage('Deploy') { - if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') { + if (env.BRANCH_NAME == 'main') { + parallel( + 'Deploy Docs': { + if (env.DOCS_DEPLOY_ENABLED == 'yes') { node('CPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/deploy-docs") { - sh( - script: """ - set -eux - retry() { - local max_retries=\$1 - shift - local n=0 - local backoff_max=30 - until [ "\$n" -ge \$max_retries ] - do - "\$@" && break - n=\$((n+1)) - if [ "\$n" -eq \$max_retries ]; then - echo "failed to update after attempt \$n / \$max_retries, giving up" - exit 1 - fi - - WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') - echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" - sleep \$WAIT - done - } - - retry 3 aws s3 cp --no-progress s3://${s3_prefix}/docs/docs.tgz docs.tgz - md5sum docs.tgz - """, - label: 'Download artifacts from S3', - ) - - deploy_docs() + timeout(time: max_time, unit: 'MINUTES') { + sh( + script: """ + set -eux + retry() { + local max_retries=\$1 + shift + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT + done + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/docs/docs.tgz docs.tgz + md5sum docs.tgz + """, + label: 'Download artifacts from S3', + ) + + deploy_docs() + } } } + } else { + Utils.markStageSkippedForConditional('Deploy Docs') } - if (env.BRANCH_NAME == 'main' && env.DEPLOY_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null) { + }, + 'Upload built Docker images': { + if (env.DEPLOY_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null) { node('CPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/deploy-docker") { - try { + timeout(time: max_time, unit: 'MINUTES') { + try { + withCredentials([string( + credentialsId: 'dockerhub-tlcpackstaging-key', + variable: 'DOCKERHUB_KEY', + )]) { + sh( + script: 'docker login -u tlcpackstaging -p ${DOCKERHUB_KEY}', + label: 'Log in to Docker Hub', + ) + } + def date_Ymd_HMS = sh( + script: 'python3 -c \'import datetime; print(datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))\'', + label: 'Determine date', + returnStdout: true, + ).trim() + def tag = "${date_Ymd_HMS}-${upstream_revision.substring(0, 8)}" + update_docker(built_ci_arm, "tlcpackstaging/ci_arm:${tag}") + update_docker(built_ci_cpu, "tlcpackstaging/ci_cpu:${tag}") + update_docker(built_ci_minimal, "tlcpackstaging/ci_minimal:${tag}") + update_docker(built_ci_gpu, "tlcpackstaging/ci_gpu:${tag}") + update_docker(built_ci_hexagon, "tlcpackstaging/ci_hexagon:${tag}") + update_docker(built_ci_i386, "tlcpackstaging/ci_i386:${tag}") + update_docker(built_ci_lint, "tlcpackstaging/ci_lint:${tag}") + update_docker(built_ci_cortexm, "tlcpackstaging/ci_cortexm:${tag}") + update_docker(built_ci_wasm, "tlcpackstaging/ci_wasm:${tag}") + } finally { + sh( + script: 'docker logout', + label: 'Clean up login credentials' + ) + } + } + } + } + } else { + Utils.markStageSkippedForConditional('Upload built Docker images') + } + }, + 'Tag tlcpackstaging to tlcpack': { + if (env.DOCS_DEPLOY_ENABLED == 'yes') { + node('CPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/tag-images") { + timeout(time: max_time, unit: 'MINUTES') { withCredentials([string( - credentialsId: 'dockerhub-tlcpackstaging-key', - variable: 'DOCKERHUB_KEY', - )]) { - sh( - script: 'docker login -u tlcpackstaging -p ${DOCKERHUB_KEY}', - label: 'Log in to Docker Hub', - ) - } - def date_Ymd_HMS = sh( - script: 'python3 -c \'import datetime; print(datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))\'', - label: 'Determine date', - returnStdout: true, - ).trim() - def tag = "${date_Ymd_HMS}-${upstream_revision.substring(0, 8)}" - update_docker(built_ci_arm, "tlcpackstaging/ci_arm:${tag}") - update_docker(built_ci_cpu, "tlcpackstaging/ci_cpu:${tag}") - update_docker(built_ci_minimal, "tlcpackstaging/ci_minimal:${tag}") - update_docker(built_ci_gpu, "tlcpackstaging/ci_gpu:${tag}") - update_docker(built_ci_hexagon, "tlcpackstaging/ci_hexagon:${tag}") - update_docker(built_ci_i386, "tlcpackstaging/ci_i386:${tag}") - update_docker(built_ci_lint, "tlcpackstaging/ci_lint:${tag}") - update_docker(built_ci_cortexm, "tlcpackstaging/ci_cortexm:${tag}") - update_docker(built_ci_wasm, "tlcpackstaging/ci_wasm:${tag}") - } finally { - sh( - script: 'docker logout', - label: 'Clean up login credentials' - ) + credentialsId: 'dockerhub-tlcpack-key', + variable: 'TLCPACK_TOKEN', + )]) { + try { + sh( + script: 'echo $TLCPACK_TOKEN | docker login --username octomldriazati --password-stdin', + label: 'Log in to Docker Hub' + ) + if (ci_arm.contains("tlcpackstaging")) { + // Push image to tlcpack + def tag = ci_arm.split(":")[1] + sh( + script: """ + set -eux + docker pull tlcpackstaging/ci_arm:${tag} + docker tag tlcpackstaging/ci_arm:${tag} tlcpack/ci-arm:${tag} + docker push tlcpack/ci-arm:${tag} + """, + label: 'Tag tlcpackstaging/ci_arm image to tlcpack', + ) + } + if (ci_cpu.contains("tlcpackstaging")) { + // Push image to tlcpack + def tag = ci_cpu.split(":")[1] + sh( + script: """ + set -eux + docker pull tlcpackstaging/ci_cpu:${tag} + docker tag tlcpackstaging/ci_cpu:${tag} tlcpack/ci-cpu:${tag} + docker push tlcpack/ci-cpu:${tag} + """, + label: 'Tag tlcpackstaging/ci_cpu image to tlcpack', + ) + } + if (ci_minimal.contains("tlcpackstaging")) { + // Push image to tlcpack + def tag = ci_minimal.split(":")[1] + sh( + script: """ + set -eux + docker pull tlcpackstaging/ci_minimal:${tag} + docker tag tlcpackstaging/ci_minimal:${tag} tlcpack/ci-minimal:${tag} + docker push tlcpack/ci-minimal:${tag} + """, + label: 'Tag tlcpackstaging/ci_minimal image to tlcpack', + ) + } + if (ci_gpu.contains("tlcpackstaging")) { + // Push image to tlcpack + def tag = ci_gpu.split(":")[1] + sh( + script: """ + set -eux + docker pull tlcpackstaging/ci_gpu:${tag} + docker tag tlcpackstaging/ci_gpu:${tag} tlcpack/ci-gpu:${tag} + docker push tlcpack/ci-gpu:${tag} + """, + label: 'Tag tlcpackstaging/ci_gpu image to tlcpack', + ) + } + if (ci_hexagon.contains("tlcpackstaging")) { + // Push image to tlcpack + def tag = ci_hexagon.split(":")[1] + sh( + script: """ + set -eux + docker pull tlcpackstaging/ci_hexagon:${tag} + docker tag tlcpackstaging/ci_hexagon:${tag} tlcpack/ci-hexagon:${tag} + docker push tlcpack/ci-hexagon:${tag} + """, + label: 'Tag tlcpackstaging/ci_hexagon image to tlcpack', + ) + } + if (ci_i386.contains("tlcpackstaging")) { + // Push image to tlcpack + def tag = ci_i386.split(":")[1] + sh( + script: """ + set -eux + docker pull tlcpackstaging/ci_i386:${tag} + docker tag tlcpackstaging/ci_i386:${tag} tlcpack/ci-i386:${tag} + docker push tlcpack/ci-i386:${tag} + """, + label: 'Tag tlcpackstaging/ci_i386 image to tlcpack', + ) + } + if (ci_lint.contains("tlcpackstaging")) { + // Push image to tlcpack + def tag = ci_lint.split(":")[1] + sh( + script: """ + set -eux + docker pull tlcpackstaging/ci_lint:${tag} + docker tag tlcpackstaging/ci_lint:${tag} tlcpack/ci-lint:${tag} + docker push tlcpack/ci-lint:${tag} + """, + label: 'Tag tlcpackstaging/ci_lint image to tlcpack', + ) + } + if (ci_cortexm.contains("tlcpackstaging")) { + // Push image to tlcpack + def tag = ci_cortexm.split(":")[1] + sh( + script: """ + set -eux + docker pull tlcpackstaging/ci_cortexm:${tag} + docker tag tlcpackstaging/ci_cortexm:${tag} tlcpack/ci-cortexm:${tag} + docker push tlcpack/ci-cortexm:${tag} + """, + label: 'Tag tlcpackstaging/ci_cortexm image to tlcpack', + ) + } + if (ci_wasm.contains("tlcpackstaging")) { + // Push image to tlcpack + def tag = ci_wasm.split(":")[1] + sh( + script: """ + set -eux + docker pull tlcpackstaging/ci_wasm:${tag} + docker tag tlcpackstaging/ci_wasm:${tag} tlcpack/ci-wasm:${tag} + docker push tlcpack/ci-wasm:${tag} + """, + label: 'Tag tlcpackstaging/ci_wasm image to tlcpack', + ) + } + } finally { + sh( + script: 'docker logout', + label: 'Clean up login credentials' + ) + } + } } } } + } else { + Utils.markStageSkippedForConditional('Tag tlcpackstaging to tlcpack') + } + }, + ) } } } diff --git a/ci/jenkins/Deploy.groovy.j2 b/ci/jenkins/Deploy.groovy.j2 index 636fcdb5a96f..08516da41b9f 100644 --- a/ci/jenkins/Deploy.groovy.j2 +++ b/ci/jenkins/Deploy.groovy.j2 @@ -82,17 +82,21 @@ def deploy_docs() { def deploy() { stage('Deploy') { - if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') { - node('CPU') { - ws({{ m.per_exec_ws('tvm/deploy-docs') }}) { + if (env.BRANCH_NAME == 'main') { + parallel( + {% call m.deploy_step( + name="Deploy Docs", + feature_flag="env.DOCS_DEPLOY_ENABLED == 'yes'", + ws="tvm/deploy-docs", + ) %} {{ m.download_artifacts(tag='docs', filenames=["docs.tgz"]) }} deploy_docs() - } - } - } - if (env.BRANCH_NAME == 'main' && env.DEPLOY_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null) { - node('CPU') { - ws({{ m.per_exec_ws('tvm/deploy-docker') }}) { + {% endcall %} + {% call m.deploy_step( + name="Upload built Docker images", + feature_flag="env.DEPLOY_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null", + ws="tvm/deploy-docker", + ) %} try { withCredentials([string( credentialsId: 'dockerhub-tlcpackstaging-key', @@ -118,8 +122,45 @@ def deploy() { label: 'Clean up login credentials' ) } - } - } + {% endcall %} + {% call m.deploy_step( + name="Tag tlcpackstaging to tlcpack", + feature_flag="env.DOCS_DEPLOY_ENABLED == 'yes'", + ws="tvm/tag-images", + ) %} + withCredentials([string( + credentialsId: 'dockerhub-tlcpack-key', + variable: 'TLCPACK_TOKEN', + )]) { + try { + sh( + script: 'echo $TLCPACK_TOKEN | docker login --username octomldriazati --password-stdin', + label: 'Log in to Docker Hub' + ) + {% for image in images %} + if ({{ image.name }}.contains("tlcpackstaging")) { + // Push image to tlcpack + def tag = {{ image.name }}.split(":")[1] + sh( + script: """ + set -eux + docker pull tlcpackstaging/{{ image.name }}:${tag} + docker tag tlcpackstaging/{{ image.name }}:${tag} tlcpack/{{ image.name.replace("_", "-") }}:${tag} + docker push tlcpack/{{ image.name.replace("_", "-") }}:${tag} + """, + label: 'Tag tlcpackstaging/{{ image.name }} image to tlcpack', + ) + } + {% endfor %} + } finally { + sh( + script: 'docker logout', + label: 'Clean up login credentials' + ) + } + } + {% endcall %} + ) } } } diff --git a/ci/jenkins/macros.j2 b/ci/jenkins/macros.j2 index 1ecfa56d0c35..dbd6ac551db4 100644 --- a/ci/jenkins/macros.j2 +++ b/ci/jenkins/macros.j2 @@ -159,6 +159,22 @@ retry() { } {% endmacro %} +{% macro deploy_step(name, feature_flag, ws) %} + '{{ name }}': { + if ({{ feature_flag }}) { + node('CPU') { + ws({{ per_exec_ws(ws) }}) { + timeout(time: max_time, unit: 'MINUTES') { + {{ caller() | indent(width=10) | trim }} + } + } + } + } else { + Utils.markStageSkippedForConditional('{{ name }}') + } + }, +{% endmacro %} + {% macro upload_artifacts(tag, filenames, folders=None) %} sh( script: """