[ci] Split out C++ unittests

This makes C++ unittests follow the normal flow of build -> upload artifacts -> download and run tests. To simplify the changes there is a new utility for interacting with S3.
apache · Nov 18, 2022 · bd260ee · bd260ee
1 parent 01a4725
commit bd260ee
Show file tree

Hide file tree

Showing 9 changed files with 579 additions and 1,312 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
diff --git a/ci/jenkins/Build.groovy.j2 b/ci/jenkins/Build.groovy.j2
@@ -1,7 +1,7 @@
 def ci_setup(image) {
   sh (
-    script: "${docker_run} ${image} ./tests/scripts/task_ci_setup.sh",
-    label: 'Set up CI environment',
+    script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
+    label: 'Clean up old workspace',
   )
 }
 
@@ -19,17 +19,40 @@ def fsim_test(image) {
   )
 }
 
-def cmake_build(image, path, make_flag) {
+def make_standalone_crt(image, build_dir) {
   sh (
-    script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod",
-    label: 'Run cmake build',
+    script: """
+      set -eux
+      ${docker_run} ${image} python3 ./tests/scripts/task_build.py \
+        --sccache-bucket tvm-sccache-prod \
+        --cmake-target standalone_crt \
+        --build-dir build
+      ${docker_run} ${image} python3 ./tests/scripts/task_build.py \
+        --sccache-bucket tvm-sccache-prod \
+        --cmake-target crttest \
+        --build-dir build
+      """,
+    label: 'Make standalone CRT',
+  )
+}
+
+def make_cpp_tests(image, build_dir) {
+  sh (
+    script: """
+      set -eux
+      ${docker_run} ${image} python3 ./tests/scripts/task_build.py \
+        --sccache-bucket tvm-sccache-prod \
+        --cmake-target cpptest \
+        --build-dir ${build_dir}
+      """,
+    label: 'Make C++ tests',
   )
 }
 
-def cpp_unittest(image) {
+def cmake_build(image, path, make_flag) {
   sh (
-    script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh",
-    label: 'Build and run C++ tests',
+    script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod",
+    label: 'Run cmake build',
   )
 }
 
@@ -51,26 +74,13 @@ def add_hexagon_permissions() {
   {% endfor %}
 }
 
-// Run make. First try to do an incremental make from a previous workspace in hope to
-// accelerate the compilation. If something is wrong, clean the workspace and then
-// build from scratch.
-def make(docker_type, path, make_flag) {
-  timeout(time: max_time, unit: 'MINUTES') {
-    try {
-      cmake_build(docker_type, path, make_flag)
-    } catch (hudson.AbortException ae) {
-      // script exited due to user abort, directly throw instead of retry
-      if (ae.getMessage().contains('script returned exit code 143')) {
-        throw ae
-      }
-      echo 'Incremental compilation failed. Fall back to build from scratch'
-      sh (
-        script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}",
-        label: 'Clear old cmake workspace',
-      )
-      cmake_build(docker_type, path, make_flag)
-    }
-  }
+def add_crttest_permissions() {
+  {% for file in crttest %}
+  sh(
+    script: 'chmod +x {{ file }}',
+    label: 'Add execute permissions for crttest',
+  )
+  {% endfor %}
 }
 
 
@@ -89,13 +99,16 @@ stage('Build') {
       docker_image='ci_gpu',
     ) %}
     sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
-    make("${ci_gpu} --no-gpu", 'build', '-j2')
-    {{ m.upload_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }}
+    cmake_build("${ci_gpu} --no-gpu", 'build', '-j2')
+    make_standalone_crt("${ci_gpu} --no-gpu", 'build')
+    {{ m.upload_artifacts(tag='gpu', filenames=tvm_multilib + microtvm_template_projects + crttest + standalone_crt) }}
 
     // compiler test
-    sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build2"
-    make("${ci_gpu} --no-gpu", 'build2', '-j2')
-    {{ m.upload_artifacts(tag='gpu2', filenames=tvm_multilib) }}
+    sh "rm -rf build"
+    sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build"
+    cmake_build("${ci_gpu} --no-gpu", 'build', '-j2')
+    make_standalone_crt("${ci_gpu} --no-gpu", 'build')
+    {{ m.upload_artifacts(tag='gpu2', filenames=tvm_lib + crttest + standalone_crt) }}
   {% endcall %}
 
   {% call m.build_step(
@@ -109,8 +122,10 @@ stage('Build') {
       script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build",
       label: 'Create CPU cmake config',
     )
-    make(ci_cpu, 'build', '-j2')
-    {{ m.upload_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }}
+    cmake_build(ci_cpu, 'build', '-j2')
+    make_standalone_crt(ci_cpu, 'build')
+    make_cpp_tests(ci_cpu, 'build')
+    {{ m.upload_artifacts(tag='cpu', filenames=tvm_multilib_tsim + crttest + cpptest + standalone_crt) }}
     ci_setup(ci_cpu)
     // sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh"
     // TODO(@jroesch): need to resolve CI issue will turn back on in follow up patch
@@ -128,8 +143,9 @@ stage('Build') {
       script: "${docker_run} ${ci_minimal} ./tests/scripts/task_config_build_minimal.sh build",
       label: 'Create CPU minimal cmake config',
     )
-    make(ci_minimal, 'build', '-j2')
-    {{ m.upload_artifacts(tag='cpu-minimal', filenames=tvm_lib) }}
+    cmake_build(ci_minimal, 'build', '-j2')
+    make_cpp_tests(ci_minimal, 'build')
+    {{ m.upload_artifacts(tag='cpu-minimal', filenames=tvm_lib + cpptest) }}
   {% endcall %}
 
   {% call m.build_step(
@@ -143,7 +159,9 @@ stage('Build') {
       script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build",
       label: 'Create WASM cmake config',
     )
-    make(ci_wasm, 'build', '-j2')
+    cmake_build(ci_wasm, 'build', '-j2')
+    make_standalone_crt(ci_wasm, 'build')
+    make_cpp_tests(ci_wasm, 'build')
     cpp_unittest(ci_wasm)
     ci_setup(ci_wasm)
     sh (
@@ -163,8 +181,10 @@ stage('Build') {
       script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build",
       label: 'Create i386 cmake config',
     )
-    make(ci_i386, 'build', '-j2')
-    {{ m.upload_artifacts(tag='i386', filenames=tvm_multilib_tsim) }}
+    cmake_build(ci_i386, 'build', '-j2')
+    make_standalone_crt(ci_i386, 'build')
+    make_cpp_tests(ci_i386, 'build')
+    {{ m.upload_artifacts(tag='i386', filenames=tvm_multilib_tsim + standalone_crt + crttest + cpptest) }}
   {% endcall %}
 
   {% call m.build_step(
@@ -178,8 +198,10 @@ stage('Build') {
       script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build",
       label: 'Create ARM cmake config',
     )
-    make(ci_arm, 'build', '-j4')
-    {{ m.upload_artifacts(tag='arm', filenames=tvm_multilib) }}
+    cmake_build(ci_arm, 'build', '-j4')
+    make_standalone_crt(ci_arm, 'build')
+    make_cpp_tests(ci_arm, 'build')
+    {{ m.upload_artifacts(tag='arm', filenames=tvm_multilib + cpptest + standalone_crt) }}
   {% endcall %}
 
   {% call m.build_step(
@@ -193,8 +215,10 @@ stage('Build') {
       script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_config_build_cortexm.sh build",
       label: 'Create Cortex-M cmake config',
     )
-    make(ci_cortexm, 'build', '-j2')
-    {{ m.upload_artifacts(tag='cortexm', filenames=tvm_lib, folders=microtvm_template_projects) }}
+    cmake_build(ci_cortexm, 'build', '-j2')
+    make_standalone_crt(ci_cortexm, 'build')
+    make_cpp_tests(ci_cortexm, 'build')
+    {{ m.upload_artifacts(tag='cortexm', filenames=tvm_lib + crttest + standalone_crt + cpptest + microtvm_template_projects) }}
   {% endcall %}
 
   {% call m.build_step(
@@ -208,12 +232,13 @@ stage('Build') {
       script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build",
       label: 'Create Hexagon cmake config',
     )
-    make(ci_hexagon, 'build', '-j2')
+    cmake_build(ci_hexagon, 'build', '-j2')
+    make_cpp_tests(ci_hexagon, 'build')
     sh (
       script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
       label: 'Build Hexagon API',
     )
-    {{ m.upload_artifacts(tag='hexagon', filenames=tvm_lib, folders=hexagon_api) }}
+    {{ m.upload_artifacts(tag='hexagon', filenames=tvm_lib + cpptest + hexagon_api) }}
   {% endcall %}
 
   {% call m.build_step(
@@ -227,8 +252,10 @@ stage('Build') {
       script: "${docker_run} ${ci_riscv} ./tests/scripts/task_config_build_riscv.sh build",
       label: 'Create RISC-V cmake config',
     )
-    make(ci_riscv, 'build', '-j2')
-    {{ m.upload_artifacts(tag='riscv', filenames=tvm_lib, folders=microtvm_template_projects) }}
+    cmake_build(ci_riscv, 'build', '-j2')
+    make_standalone_crt(ci_riscv, 'build')
+    make_cpp_tests(ci_riscv, 'build')
+    {{ m.upload_artifacts(tag='riscv', filenames=tvm_lib + crttest + cpptest + microtvm_template_projects) }}
   {% endcall %}
 
   )

diff --git a/ci/jenkins/Jenkinsfile.j2 b/ci/jenkins/Jenkinsfile.j2
@@ -93,12 +93,16 @@ rebuild_docker_images = false
 
 // Filenames for stashing between build and test steps
 {% set tvm_runtime = ['build/libtvm_runtime.so', 'build/config.cmake'] %}
+{% set crttest = ['build/crttest'] %}
+{% set cpptest = ['build/cpptest'] %}
 {% set tvm_lib = ['build/libtvm.so'] + tvm_runtime %}
 {% set tvm_multilib = ['build/libtvm.so', 'build/libvta_fsim.so'] + tvm_runtime %}
 {% set tvm_multilib_tsim = ['build/libvta_tsim.so'] + tvm_multilib %}
 {% set microtvm_template_projects = ['build/microtvm_template_projects',] %}
 {% set hexagon_api = ['build/hexagon_api_output',] %}
-s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
+{% set standalone_crt = ['build/standalone_crt'] %}
+s3_bucket = 'tvm-jenkins-artifacts-prod'
+s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
 
 // Jenkins script root directory
 jenkins_scripts_root = "ci/scripts/jenkins"

diff --git a/ci/jenkins/Test.groovy.j2 b/ci/jenkins/Test.groovy.j2
@@ -1,5 +1,12 @@
 {% set test_method_names = [] %}
 
+def cpp_unittest(image) {
+  sh (
+    script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh",
+    label: 'Run C++ tests',
+  )
+}
+
 // We have to do this whacky split of the code from where it's used since the
 // JVM limits method length to 64k and we easily exceed that with all this
 // autogenerated code. This makes it so each test step is in its own method so
@@ -14,11 +21,19 @@
   test_method_names=test_method_names,
 ) %}
   {% if shard_index == 1 %}
-  {{ m.download_artifacts(tag='gpu2', filenames=tvm_multilib) }}
-  cpp_unittest(ci_gpu)
+  {{ m.download_artifacts(tag='gpu2', filenames=tvm_multilib + crttest + standalone_crt) }}
+  sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build"
+  // These require a GPU to finish the build (i.e. CUDA needs to be load-able)
+  make_standalone_crt(ci_gpu, 'build')
+  // make_cpp_tests(ci_gpu, 'build')
+  // cpp_unittest(ci_gpu)
 
-  {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }}
+  sh "rm -rf build"
+  {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib + microtvm_template_projects + crttest + standalone_crt) }}
   ci_setup(ci_gpu)
+  sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
+  make_standalone_crt(ci_gpu, 'build')
+  make_cpp_tests(ci_gpu, 'build')
   cpp_unittest(ci_gpu)
   {% else %}
   {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }}
@@ -48,7 +63,7 @@
   docker_image="ci_cpu",
   test_method_names=test_method_names,
 ) %}
-  {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }}
+  {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim + standalone_crt) }}
   ci_setup(ci_cpu)
   sh (
     script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh",
@@ -64,7 +79,7 @@
   docker_image="ci_i386",
   test_method_names=test_method_names,
 ) %}
-  {{ m.download_artifacts(tag='i386', filenames=tvm_multilib) }}
+  {{ m.download_artifacts(tag='i386', filenames=tvm_multilib_tsim + standalone_crt + crttest + cpptest) }}
   ci_setup(ci_i386)
   {% if shard_index == 1 %}
   cpp_unittest(ci_i386)
@@ -87,8 +102,7 @@
   test_method_names=test_method_names,
   num_shards=8,
 ) %}
-  {{ m.download_artifacts(tag='hexagon', filenames=tvm_lib, folders=hexagon_api) }}
-  add_hexagon_permissions()
+  {{ m.download_artifacts(tag='hexagon', filenames=tvm_lib + hexagon_api + cpptest) }}
   ci_setup(ci_hexagon)
   {% if shard_index == 1 %}
   cpp_unittest(ci_hexagon)
@@ -107,7 +121,7 @@
   docker_image="ci_arm",
   test_method_names=test_method_names,
 ) %}
-  {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }}
+  {{ m.download_artifacts(tag='arm', filenames=tvm_multilib + standalone_crt) }}
   ci_setup(ci_arm)
   python_unittest(ci_arm)
   sh (
@@ -156,7 +170,7 @@
   num_shards=2,
   test_method_names=test_method_names,
 ) %}
-  {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }}
+  {{ m.download_artifacts(tag='arm', filenames=tvm_multilib + cpptest) }}
   ci_setup(ci_arm)
   {% if shard_index == 1 %}
   cpp_unittest(ci_arm)
@@ -195,8 +209,7 @@
   num_shards=12,
   test_method_names=test_method_names,
 ) %}
-  {{ m.download_artifacts(tag='cortexm', filenames=tvm_lib, folders=microtvm_template_projects) }}
-  add_microtvm_permissions()
+  {{ m.download_artifacts(tag='cortexm', filenames=tvm_lib + microtvm_template_projects + cpptest + standalone_crt) }}
   ci_setup(ci_cortexm)
   {% if shard_index == 1%}
   cpp_unittest(ci_cortexm)
@@ -219,8 +232,7 @@
   num_shards=1,
   test_method_names=test_method_names,
 ) %}
-  {{ m.download_artifacts(tag='riscv', filenames=tvm_lib, folders=microtvm_template_projects) }}
-  add_microtvm_permissions()
+  {{ m.download_artifacts(tag='riscv', filenames=tvm_lib + microtvm_template_projects + cpptest) }}
   ci_setup(ci_riscv)
   {% if shard_index == 1%}
   cpp_unittest(ci_cortexm)
@@ -239,7 +251,7 @@ def run_unittest_minimal() {
       platform="minimal",
       docker_image="ci_minimal",
     ) %}
-      {{ m.download_artifacts(tag='cpu-minimal', filenames=tvm_lib) }}
+      {{ m.download_artifacts(tag='cpu-minimal', filenames=tvm_lib + cpptest) }}
       cpp_unittest(ci_minimal)
       python_unittest(ci_minimal)
   {% endcall %}
@@ -266,7 +278,7 @@ stage('Test') {
     platform="cpu",
     docker_image="ci_cpu",
   ) %}
-    {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }}
+    {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim + cpptest + standalone_crt) }}
     ci_setup(ci_cpu)
     cpp_unittest(ci_cpu)
     python_unittest(ci_cpu)
@@ -296,8 +308,7 @@ stage('Test') {
         ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) {
           init_git()
           docker_init(ci_gpu)
-          {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }}
-          add_microtvm_permissions()
+          {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib + microtvm_template_projects + standalone_crt) }}
           timeout(time: 180, unit: 'MINUTES') {
             ci_setup(ci_gpu)
             sh (
@@ -307,7 +318,7 @@ stage('Test') {
           }
           {{ m.upload_artifacts(tag='docs', filenames=["docs.tgz"]) }}
           sh(
-            script: "aws s3 cp --no-progress _docs s3://${s3_prefix}/docs --recursive",
+            script: "aws s3 cp --no-progress _docs s3://${s3_bucket}/${s3_prefix}/docs --recursive",
             label: 'Upload docs to S3',
           )
         }