From 1c1dd104bafe4ffe4e4c69f2f8b9a99bc765fdf1 Mon Sep 17 00:00:00 2001 From: driazati <9407960+driazati@users.noreply.github.com> Date: Fri, 27 May 2022 12:59:17 -0700 Subject: [PATCH] [skip ci] Revert "[skip ci][ci][docker] Prune all non-relevant images (#11491)" (#11496) --- Jenkinsfile | 88 ++++------------------------------- jenkins/Build.groovy.j2 | 7 --- jenkins/DockerBuild.groovy.j2 | 8 ---- jenkins/Lint.groovy.j2 | 1 - jenkins/Prepare.groovy.j2 | 23 ++------- jenkins/Test.groovy.j2 | 15 +----- jenkins/macros.j2 | 9 ++-- 7 files changed, 17 insertions(+), 134 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7f27fb67edff4..b5cdb06db9e85 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-05-27T14:44:51.598735 +// Generated at 2022-05-27T14:45:01.071408 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -108,7 +108,11 @@ def per_exec_ws(folder) { def init_git() { checkout scm - + // Clear out all Docker images that aren't going to be used + sh( + script: "docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}' | { grep -vE '${ci_arm}|${ci_cpu}|${ci_gpu}|${ci_hexagon}|${ci_i386}|${ci_lint}|${ci_qemu}|${ci_wasm}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }", + label: 'Clean old Docker images', + ) // Add more info about job node sh ( script: './tests/scripts/task_show_node_info.sh', @@ -156,23 +160,6 @@ def init_git() { ) } -def docker_init(image) { - // Clear out all Docker images that aren't going to be used - sh( - script: """ - set -eux - docker image ls --all - IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}') - - echo -e "Found images:\\n\$IMAGES" - echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; } - - docker image ls --all - """, - label: 'Clean old Docker images', - ) -} - def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', @@ -334,7 +321,6 @@ def build_docker_images() { parallel 'ci-lint': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_lint') } @@ -342,7 +328,6 @@ def build_docker_images() { }, 'ci-cpu': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_cpu') } @@ -350,7 +335,6 @@ def build_docker_images() { }, 'ci-gpu': { node('GPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_gpu') } @@ -358,7 +342,6 @@ def build_docker_images() { }, 'ci-qemu': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_qemu') } @@ -366,7 +349,6 @@ def build_docker_images() { }, 'ci-i386': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_i386') } @@ -374,7 +356,6 @@ def build_docker_images() { }, 'ci-arm': { node('ARM') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_arm') } @@ -382,7 +363,6 @@ def build_docker_images() { }, 'ci-wasm': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_wasm') } @@ -390,7 +370,6 @@ def build_docker_images() { }, 'ci-hexagon': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_hexagon') } @@ -445,7 +424,6 @@ def lint() { 'Lint 1 of 2': { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") { - docker_init(ci_lint) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -463,7 +441,6 @@ def lint() { 'Lint 2 of 2': { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") { - docker_init(ci_lint) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -541,7 +518,6 @@ stage('Build') { if (!skip_ci) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") { - docker_init(ci_gpu) init_git() sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" make("${ci_gpu} --no-gpu", 'build', '-j2') @@ -588,7 +564,6 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") { - docker_init(ci_cpu) init_git() sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", @@ -628,7 +603,6 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") { - docker_init(ci_wasm) init_git() sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", @@ -653,7 +627,6 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") { - docker_init(ci_386) init_git() sh ( script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build", @@ -687,7 +660,6 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") { - docker_init(ci_arm) init_git() sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build", @@ -719,7 +691,6 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-qemu") { - docker_init(ci_qemu) init_git() sh ( script: "${docker_run} ${ci_qemu} ./tests/scripts/task_config_build_qemu.sh build", @@ -750,7 +721,6 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-hexagon") { - docker_init(ci_hexagon) init_git() sh ( script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build", @@ -795,7 +765,6 @@ def shard_run_unittest_GPU_1_of_3() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -861,7 +830,6 @@ def shard_run_unittest_GPU_2_of_3() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -913,7 +881,6 @@ def shard_run_unittest_GPU_3_of_3() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -962,7 +929,6 @@ def shard_run_integration_CPU_1_of_6() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { - docker_init(ci_cpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1008,7 +974,6 @@ def shard_run_integration_CPU_2_of_6() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { - docker_init(ci_cpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1054,7 +1019,6 @@ def shard_run_integration_CPU_3_of_6() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { - docker_init(ci_cpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1100,7 +1064,6 @@ def shard_run_integration_CPU_4_of_6() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { - docker_init(ci_cpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1146,7 +1109,6 @@ def shard_run_integration_CPU_5_of_6() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { - docker_init(ci_cpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1192,7 +1154,6 @@ def shard_run_integration_CPU_6_of_6() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { try { - docker_init(ci_cpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1239,7 +1200,6 @@ def shard_run_python_i386_1_of_5() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { - docker_init(ci_i386) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1286,7 +1246,6 @@ def shard_run_python_i386_2_of_5() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { - docker_init(ci_i386) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1332,7 +1291,6 @@ def shard_run_python_i386_3_of_5() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { - docker_init(ci_i386) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1378,7 +1336,6 @@ def shard_run_python_i386_4_of_5() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { - docker_init(ci_i386) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1424,7 +1381,6 @@ def shard_run_python_i386_5_of_5() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { try { - docker_init(ci_i386) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1471,7 +1427,6 @@ def shard_run_test_Hexagon_1_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1516,7 +1471,6 @@ def shard_run_test_Hexagon_2_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1560,7 +1514,6 @@ def shard_run_test_Hexagon_3_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1604,7 +1557,6 @@ def shard_run_test_Hexagon_4_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1648,7 +1600,6 @@ def shard_run_test_Hexagon_5_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1692,7 +1643,6 @@ def shard_run_test_Hexagon_6_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1736,7 +1686,6 @@ def shard_run_test_Hexagon_7_of_7() { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { try { - docker_init(ci_hexagon) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1781,7 +1730,6 @@ def shard_run_integration_aarch64_1_of_4() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { - docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1826,7 +1774,6 @@ def shard_run_integration_aarch64_2_of_4() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { - docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1871,7 +1818,6 @@ def shard_run_integration_aarch64_3_of_4() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { - docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1916,7 +1862,6 @@ def shard_run_integration_aarch64_4_of_4() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { - docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -1962,7 +1907,6 @@ def shard_run_topi_GPU_1_of_4() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2006,7 +1950,6 @@ def shard_run_topi_GPU_2_of_4() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2050,7 +1993,6 @@ def shard_run_topi_GPU_3_of_4() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2094,7 +2036,6 @@ def shard_run_topi_GPU_4_of_4() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2139,7 +2080,6 @@ def shard_run_frontend_GPU_1_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2183,7 +2123,6 @@ def shard_run_frontend_GPU_2_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2227,7 +2166,6 @@ def shard_run_frontend_GPU_3_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2271,7 +2209,6 @@ def shard_run_frontend_GPU_4_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2315,7 +2252,6 @@ def shard_run_frontend_GPU_5_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2359,7 +2295,6 @@ def shard_run_frontend_GPU_6_of_6() { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { try { - docker_init(ci_gpu) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2404,7 +2339,6 @@ def shard_run_topi_aarch64_1_of_2() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { - docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2453,7 +2387,6 @@ def shard_run_topi_aarch64_2_of_2() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { try { - docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2503,7 +2436,6 @@ def shard_run_frontend_aarch64_1_of_2() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") { try { - docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2547,7 +2479,6 @@ def shard_run_frontend_aarch64_2_of_2() { node('ARM-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") { try { - docker_init(ci_arm) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -2717,7 +2648,6 @@ stage('Test') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") { timeout(time: max_time, unit: 'MINUTES') { try { - docker_init(ci_cpu) init_git() withEnv(['PLATFORM=cpu'], { sh( @@ -2762,7 +2692,6 @@ stage('Test') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-qemu") { timeout(time: max_time, unit: 'MINUTES') { try { - docker_init(ci_qemu) init_git() withEnv(['PLATFORM=qemu'], { sh( @@ -2807,7 +2736,6 @@ stage('Test') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-cpu") { timeout(time: max_time, unit: 'MINUTES') { try { - docker_init(ci_cpu) init_git() withEnv(['PLATFORM=cpu'], { sh( @@ -2845,7 +2773,6 @@ stage('Test') { if (!skip_ci) { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") { - docker_init(ci_gpu) init_git() sh( script: """ @@ -2887,7 +2814,8 @@ stage('Test') { }, ) } -}/* +} +/* stage('Build packages') { parallel 'conda CPU': { node('CPU') { diff --git a/jenkins/Build.groovy.j2 b/jenkins/Build.groovy.j2 index 7e19ce34e71ef..4b0b4ae2e2c80 100644 --- a/jenkins/Build.groovy.j2 +++ b/jenkins/Build.groovy.j2 @@ -62,7 +62,6 @@ stage('Build') { if (!skip_ci) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-gpu') }}) { - docker_init(ci_gpu) init_git() sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" make("${ci_gpu} --no-gpu", 'build', '-j2') @@ -80,7 +79,6 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-cpu') }}) { - docker_init(ci_cpu) init_git() sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", @@ -104,7 +102,6 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-wasm') }}) { - docker_init(ci_wasm) init_git() sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", @@ -129,7 +126,6 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-i386') }}) { - docker_init(ci_386) init_git() sh ( script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build", @@ -147,7 +143,6 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('ARM-SMALL') { ws({{ m.per_exec_ws('tvm/build-arm') }}) { - docker_init(ci_arm) init_git() sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build", @@ -165,7 +160,6 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-qemu') }}) { - docker_init(ci_qemu) init_git() sh ( script: "${docker_run} ${ci_qemu} ./tests/scripts/task_config_build_qemu.sh build", @@ -183,7 +177,6 @@ stage('Build') { if (!skip_ci && is_docs_only_build != 1) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-hexagon') }}) { - docker_init(ci_hexagon) init_git() sh ( script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build", diff --git a/jenkins/DockerBuild.groovy.j2 b/jenkins/DockerBuild.groovy.j2 index e9d80801a9d9c..84bb8e3e376d1 100644 --- a/jenkins/DockerBuild.groovy.j2 +++ b/jenkins/DockerBuild.groovy.j2 @@ -59,7 +59,6 @@ def build_docker_images() { parallel 'ci-lint': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_lint') } @@ -67,7 +66,6 @@ def build_docker_images() { }, 'ci-cpu': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_cpu') } @@ -75,7 +73,6 @@ def build_docker_images() { }, 'ci-gpu': { node('GPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_gpu') } @@ -83,7 +80,6 @@ def build_docker_images() { }, 'ci-qemu': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_qemu') } @@ -91,7 +87,6 @@ def build_docker_images() { }, 'ci-i386': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_i386') } @@ -99,7 +94,6 @@ def build_docker_images() { }, 'ci-arm': { node('ARM') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_arm') } @@ -107,7 +101,6 @@ def build_docker_images() { }, 'ci-wasm': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_wasm') } @@ -115,7 +108,6 @@ def build_docker_images() { }, 'ci-hexagon': { node('CPU') { timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') init_git() build_image('ci_hexagon') } diff --git a/jenkins/Lint.groovy.j2 b/jenkins/Lint.groovy.j2 index 40dad3aef7be3..61c13cd407d02 100644 --- a/jenkins/Lint.groovy.j2 +++ b/jenkins/Lint.groovy.j2 @@ -6,7 +6,6 @@ def lint() { num_shards=2, node='CPU-SMALL', ws='tvm/lint', - docker_image='ci_lint', ) %} sh ( diff --git a/jenkins/Prepare.groovy.j2 b/jenkins/Prepare.groovy.j2 index 2900775f49452..b4db7de63bd15 100644 --- a/jenkins/Prepare.groovy.j2 +++ b/jenkins/Prepare.groovy.j2 @@ -6,7 +6,11 @@ def per_exec_ws(folder) { def init_git() { checkout scm - + // Clear out all Docker images that aren't going to be used + sh( + script: "docker image ls --all --format {% raw %}'{{.Repository}}:{{.Tag}} {{.ID}}'{% endraw %} | { grep -vE '{% for image in images %}{% raw %}${{% endraw %}{{ image.name }}{% raw %}}{% endraw %}{% if not loop.last %}|{% endif %}{% endfor %}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }", + label: 'Clean old Docker images', + ) // Add more info about job node sh ( script: './tests/scripts/task_show_node_info.sh', @@ -54,23 +58,6 @@ def init_git() { ) } -def docker_init(image) { - // Clear out all Docker images that aren't going to be used - sh( - script: """ - set -eux - docker image ls --all - IMAGES=\$(docker image ls --all --format {% raw %}'{{.Repository}}:{{.Tag}} {{.ID}}'{% endraw %}) - - echo -e "Found images:\\n\$IMAGES" - echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; } - - docker image ls --all - """, - label: 'Clean old Docker images', - ) -} - def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/jenkins/Test.groovy.j2 b/jenkins/Test.groovy.j2 index 9f949ae717c2a..a08c50905a056 100644 --- a/jenkins/Test.groovy.j2 +++ b/jenkins/Test.groovy.j2 @@ -10,7 +10,6 @@ node="GPU", ws="tvm/ut-python-gpu", platform="gpu", - docker_image="ci_gpu", test_method_names=test_method_names, ) %} {% if shard_index == 1 %} @@ -45,7 +44,6 @@ num_shards=6, ws="tvm/integration-python-cpu", platform="cpu", - docker_image="ci_cpu", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} @@ -61,7 +59,6 @@ num_shards=5, ws="tvm/integration-python-i386", platform="i386", - docker_image="ci_i386", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='i386', filenames=tvm_multilib) }} @@ -81,7 +78,6 @@ node="CPU-SMALL", ws="tvm/test-hexagon", platform="hexagon", - docker_image="ci_hexagon", test_method_names=test_method_names, num_shards=7, ) %} @@ -102,7 +98,6 @@ node="ARM-SMALL", ws="tvm/ut-python-arm", platform="arm", - docker_image="ci_arm", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} @@ -119,7 +114,6 @@ num_shards=4, ws="tvm/topi-python-gpu", platform="gpu", - docker_image="ci_gpu", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} @@ -135,7 +129,6 @@ num_shards=6, ws="tvm/frontend-python-gpu", platform="gpu", - docker_image="ci_gpu", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} @@ -150,7 +143,6 @@ node="ARM-SMALL", ws="tvm/ut-python-arm", platform="arm", - docker_image="ci_arm", num_shards=2, test_method_names=test_method_names, ) %} @@ -171,7 +163,6 @@ node="ARM-SMALL", ws="tvm/frontend-python-arm", platform="arm", - docker_image="ci_arm", num_shards=2, test_method_names=test_method_names, ) %} @@ -200,7 +191,6 @@ stage('Test') { node="CPU-SMALL", ws="tvm/ut-python-cpu", platform="cpu", - docker_image="ci_cpu", ) %} {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} ci_setup(ci_cpu) @@ -217,7 +207,6 @@ stage('Test') { node="CPU-SMALL", ws="tvm/test-qemu", platform="qemu", - docker_image="ci_qemu", ) %} {{ m.download_artifacts(tag='qemu', filenames=tvm_lib, folders=microtvm_template_projects) }} add_microtvm_permissions() @@ -237,7 +226,6 @@ stage('Test') { node="CPU-SMALL", ws="tvm/frontend-python-cpu", platform="cpu", - docker_image="ci_cpu", ) %} {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib) }} ci_setup(ci_cpu) @@ -250,7 +238,6 @@ stage('Test') { if (!skip_ci) { node('GPU') { ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) { - docker_init(ci_gpu) init_git() {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} add_microtvm_permissions() @@ -269,4 +256,4 @@ stage('Test') { }, ) } -} \ No newline at end of file +} diff --git a/jenkins/macros.j2 b/jenkins/macros.j2 index 5a641b73fea84..1c649e31fabfd 100644 --- a/jenkins/macros.j2 +++ b/jenkins/macros.j2 @@ -19,7 +19,7 @@ "workspace/exec_${env.EXECUTOR_NUMBER}/{{ folder }}" {%- endmacro -%} -{% macro sharded_test_step(name, num_shards, node, ws, docker_image, platform, test_method_names) %} +{% macro sharded_test_step(name, num_shards, node, ws, platform, test_method_names) %} {% for shard_index in range(1, num_shards + 1) %} {% set method_name = "shard_run_" + name.replace(":", "").replace(" ", "-").replace("-", "_") + "_" + shard_index|string + "_of_" + num_shards|string %} @@ -28,7 +28,6 @@ def {{ method_name }}() { node('{{ node }}') { ws({{ per_exec_ws(ws) }}) { try { - docker_init({{ docker_image }}) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -52,12 +51,11 @@ def {{ method_name }}() { {% endfor %} {% endmacro %} -{% macro sharded_lint_step(name, num_shards, docker_image, node, ws) %} +{% macro sharded_lint_step(name, num_shards, node, ws) %} {% for shard_index in range(1, num_shards + 1) %} '{{ name }} {{ shard_index }} of {{ num_shards }}': { node('{{ node }}') { ws({{ per_exec_ws(ws) }}) { - docker_init({{ docker_image }}) init_git() timeout(time: max_time, unit: 'MINUTES') { withEnv([ @@ -73,14 +71,13 @@ def {{ method_name }}() { {% endmacro %} -{% macro test_step(name, node, ws, docker_image, platform) %} +{% macro test_step(name, node, ws, platform) %} '{{ name }}': { if (!skip_ci && is_docs_only_build != 1) { node('{{ node }}') { ws({{ per_exec_ws(ws) }}) { timeout(time: max_time, unit: 'MINUTES') { try { - docker_init({{ docker_image }}) init_git() withEnv(['PLATFORM={{ platform }}'], { {{ caller() | indent(width=12) | trim }}