From cf342a5a925e11cad0ee459a7c1c6602cbb2354e Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Tue, 22 Oct 2019 11:11:45 -0700 Subject: [PATCH 1/9] split to cd and shell --- ci/docker/runtime_functions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 935406bb16a8..4f48a54affb7 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1467,7 +1467,8 @@ nightly_test_imagenet_inference() { set -ex echo $PWD cp /work/mxnet/build/cpp-package/example/imagenet_inference . - /work/mxnet/cpp-package/example/inference/unit_test_imagenet_inference.sh + cd /work/mxnet/cpp-package/example/inference/ + ./unit_test_imagenet_inference.sh } #Runs a simple MNIST training example From 079d842df46cb5de31b81e60643adbb42e49d73d Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Tue, 22 Oct 2019 11:12:51 -0700 Subject: [PATCH 2/9] comment --- tests/nightly/JenkinsfileForBinaries | 116 +++++++++++++-------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index a66159d0075b..5b43819171d1 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -62,30 +62,30 @@ core_logic: { } stage('NightlyTests'){ - parallel 'ImageClassification: GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/nt-ImageClassificationTest') { - utils.unpack_and_init('gpu', mx_lib) - utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_image_classification', true) - } - } - }, - 'ImageNet Inference: GPU': { + // parallel 'ImageClassification: GPU': { + // node(NODE_LINUX_GPU) { + // ws('workspace/nt-ImageClassificationTest') { + // utils.unpack_and_init('gpu', mx_lib) + // utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_image_classification', true) + // } + // } + // }, + parallel 'ImageNet Inference: GPU': { node(NODE_LINUX_GPU) { ws('workspace/nt-ImageInferenceTest') { utils.unpack_and_init('gpu', mx_lib_cpp_example) utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_imagenet_inference', true) } } - }, - 'KVStore_SingleNode: GPU': { - node('mxnetlinux-gpu-p3-8xlarge') { - ws('workspace/nt-KVStoreTest') { - utils.unpack_and_init('gpu', mx_lib) - utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_KVStore_singleNode', true) - } - } - }, + }//, + // 'KVStore_SingleNode: GPU': { + // node('mxnetlinux-gpu-p3-8xlarge') { + // ws('workspace/nt-KVStoreTest') { + // utils.unpack_and_init('gpu', mx_lib) + // utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_KVStore_singleNode', true) + // } + // } + // }, // https://github.com/apache/incubator-mxnet/issues/14980 /*'Test Large Tensor Size: CPU': { node(NODE_LINUX_CPU) { @@ -104,46 +104,46 @@ core_logic: { } } },*/ - 'StraightDope: Python2 Single-GPU': { - node(NODE_LINUX_GPU_P3) { - ws('workspace/straight_dope-single_gpu') { - utils.unpack_and_init('gpu', mx_lib) - utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_single_gpu_tests', true) - } - } - }, - 'StraightDope: Python2 Multi-GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/straight_dope-multi_gpu') { - utils.unpack_and_init('gpu', mx_lib) - utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_multi_gpu_tests', true) - } - } - }, - 'StraightDope: Python3 Single-GPU': { - node(NODE_LINUX_GPU_P3) { - ws('workspace/straight_dope-single_gpu') { - utils.unpack_and_init('gpu', mx_lib) - utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_single_gpu_tests', true) - } - } - }, - 'StraightDope: Python3 Multi-GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/straight_dope-multi_gpu') { - utils.unpack_and_init('gpu', mx_lib) - utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_multi_gpu_tests', true) - } - } - }, - 'Gluon estimator: GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/estimator-test-gpu') { - utils.unpack_and_init('gpu', mx_lib) - utils.docker_run('ubuntu_nightly_gpu', 'nightly_estimator', true) - } - } - } + // 'StraightDope: Python2 Single-GPU': { + // node(NODE_LINUX_GPU_P3) { + // ws('workspace/straight_dope-single_gpu') { + // utils.unpack_and_init('gpu', mx_lib) + // utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_single_gpu_tests', true) + // } + // } + // }, + // 'StraightDope: Python2 Multi-GPU': { + // node(NODE_LINUX_GPU) { + // ws('workspace/straight_dope-multi_gpu') { + // utils.unpack_and_init('gpu', mx_lib) + // utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_multi_gpu_tests', true) + // } + // } + // }, + // 'StraightDope: Python3 Single-GPU': { + // node(NODE_LINUX_GPU_P3) { + // ws('workspace/straight_dope-single_gpu') { + // utils.unpack_and_init('gpu', mx_lib) + // utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_single_gpu_tests', true) + // } + // } + // }, + // 'StraightDope: Python3 Multi-GPU': { + // node(NODE_LINUX_GPU) { + // ws('workspace/straight_dope-multi_gpu') { + // utils.unpack_and_init('gpu', mx_lib) + // utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_multi_gpu_tests', true) + // } + // } + // }, + // 'Gluon estimator: GPU': { + // node(NODE_LINUX_GPU) { + // ws('workspace/estimator-test-gpu') { + // utils.unpack_and_init('gpu', mx_lib) + // utils.docker_run('ubuntu_nightly_gpu', 'nightly_estimator', true) + // } + // } + // } } } , From 6742e9b68e7224b3ebd0a42eb062ae88b7d7103c Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Tue, 22 Oct 2019 13:19:30 -0700 Subject: [PATCH 3/9] lots of prints --- .../example/inference/unit_test_imagenet_inference.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp-package/example/inference/unit_test_imagenet_inference.sh b/cpp-package/example/inference/unit_test_imagenet_inference.sh index c645388cd419..1e97ee751a93 100755 --- a/cpp-package/example/inference/unit_test_imagenet_inference.sh +++ b/cpp-package/example/inference/unit_test_imagenet_inference.sh @@ -38,7 +38,11 @@ wget -nc https://raw.githubusercontent.com/dmlc/gluon-cv/master/gluoncv/model_zo cd ../data wget -nc http://data.mxnet.io/data/val_256_q90.rec cd .. - +echo $PWD +echo "lib path" +ls -al ../../../lib +echo "current file" +ls -al . # Running inference on imagenet. if [ "$(uname)" == "Darwin" ]; then echo ">>> INFO: FP32 real data" From b4202ab6754f63391cc846fcd6c990b68a257a9f Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Wed, 23 Oct 2019 13:47:22 -0700 Subject: [PATCH 4/9] copy binary at correct location --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 4f48a54affb7..1eea0d365f0a 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1466,7 +1466,7 @@ nightly_test_installation() { nightly_test_imagenet_inference() { set -ex echo $PWD - cp /work/mxnet/build/cpp-package/example/imagenet_inference . + cp /work/mxnet/build/cpp-package/example/imagenet_inference /work/mxnet/cpp-package/example/inference/ cd /work/mxnet/cpp-package/example/inference/ ./unit_test_imagenet_inference.sh } From 5d364887bf6fc3ff6098ade36a4c4c45eedffabf Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Wed, 23 Oct 2019 15:31:02 -0700 Subject: [PATCH 5/9] remove comments --- .../inference/unit_test_imagenet_inference.sh | 6 +- tests/nightly/JenkinsfileForBinaries | 116 +++++++++--------- 2 files changed, 59 insertions(+), 63 deletions(-) diff --git a/cpp-package/example/inference/unit_test_imagenet_inference.sh b/cpp-package/example/inference/unit_test_imagenet_inference.sh index 1e97ee751a93..c645388cd419 100755 --- a/cpp-package/example/inference/unit_test_imagenet_inference.sh +++ b/cpp-package/example/inference/unit_test_imagenet_inference.sh @@ -38,11 +38,7 @@ wget -nc https://raw.githubusercontent.com/dmlc/gluon-cv/master/gluoncv/model_zo cd ../data wget -nc http://data.mxnet.io/data/val_256_q90.rec cd .. -echo $PWD -echo "lib path" -ls -al ../../../lib -echo "current file" -ls -al . + # Running inference on imagenet. if [ "$(uname)" == "Darwin" ]; then echo ">>> INFO: FP32 real data" diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index 5b43819171d1..a66159d0075b 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -62,30 +62,30 @@ core_logic: { } stage('NightlyTests'){ - // parallel 'ImageClassification: GPU': { - // node(NODE_LINUX_GPU) { - // ws('workspace/nt-ImageClassificationTest') { - // utils.unpack_and_init('gpu', mx_lib) - // utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_image_classification', true) - // } - // } - // }, - parallel 'ImageNet Inference: GPU': { + parallel 'ImageClassification: GPU': { + node(NODE_LINUX_GPU) { + ws('workspace/nt-ImageClassificationTest') { + utils.unpack_and_init('gpu', mx_lib) + utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_image_classification', true) + } + } + }, + 'ImageNet Inference: GPU': { node(NODE_LINUX_GPU) { ws('workspace/nt-ImageInferenceTest') { utils.unpack_and_init('gpu', mx_lib_cpp_example) utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_imagenet_inference', true) } } - }//, - // 'KVStore_SingleNode: GPU': { - // node('mxnetlinux-gpu-p3-8xlarge') { - // ws('workspace/nt-KVStoreTest') { - // utils.unpack_and_init('gpu', mx_lib) - // utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_KVStore_singleNode', true) - // } - // } - // }, + }, + 'KVStore_SingleNode: GPU': { + node('mxnetlinux-gpu-p3-8xlarge') { + ws('workspace/nt-KVStoreTest') { + utils.unpack_and_init('gpu', mx_lib) + utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_KVStore_singleNode', true) + } + } + }, // https://github.com/apache/incubator-mxnet/issues/14980 /*'Test Large Tensor Size: CPU': { node(NODE_LINUX_CPU) { @@ -104,46 +104,46 @@ core_logic: { } } },*/ - // 'StraightDope: Python2 Single-GPU': { - // node(NODE_LINUX_GPU_P3) { - // ws('workspace/straight_dope-single_gpu') { - // utils.unpack_and_init('gpu', mx_lib) - // utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_single_gpu_tests', true) - // } - // } - // }, - // 'StraightDope: Python2 Multi-GPU': { - // node(NODE_LINUX_GPU) { - // ws('workspace/straight_dope-multi_gpu') { - // utils.unpack_and_init('gpu', mx_lib) - // utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_multi_gpu_tests', true) - // } - // } - // }, - // 'StraightDope: Python3 Single-GPU': { - // node(NODE_LINUX_GPU_P3) { - // ws('workspace/straight_dope-single_gpu') { - // utils.unpack_and_init('gpu', mx_lib) - // utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_single_gpu_tests', true) - // } - // } - // }, - // 'StraightDope: Python3 Multi-GPU': { - // node(NODE_LINUX_GPU) { - // ws('workspace/straight_dope-multi_gpu') { - // utils.unpack_and_init('gpu', mx_lib) - // utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_multi_gpu_tests', true) - // } - // } - // }, - // 'Gluon estimator: GPU': { - // node(NODE_LINUX_GPU) { - // ws('workspace/estimator-test-gpu') { - // utils.unpack_and_init('gpu', mx_lib) - // utils.docker_run('ubuntu_nightly_gpu', 'nightly_estimator', true) - // } - // } - // } + 'StraightDope: Python2 Single-GPU': { + node(NODE_LINUX_GPU_P3) { + ws('workspace/straight_dope-single_gpu') { + utils.unpack_and_init('gpu', mx_lib) + utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_single_gpu_tests', true) + } + } + }, + 'StraightDope: Python2 Multi-GPU': { + node(NODE_LINUX_GPU) { + ws('workspace/straight_dope-multi_gpu') { + utils.unpack_and_init('gpu', mx_lib) + utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_multi_gpu_tests', true) + } + } + }, + 'StraightDope: Python3 Single-GPU': { + node(NODE_LINUX_GPU_P3) { + ws('workspace/straight_dope-single_gpu') { + utils.unpack_and_init('gpu', mx_lib) + utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_single_gpu_tests', true) + } + } + }, + 'StraightDope: Python3 Multi-GPU': { + node(NODE_LINUX_GPU) { + ws('workspace/straight_dope-multi_gpu') { + utils.unpack_and_init('gpu', mx_lib) + utils.docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_multi_gpu_tests', true) + } + } + }, + 'Gluon estimator: GPU': { + node(NODE_LINUX_GPU) { + ws('workspace/estimator-test-gpu') { + utils.unpack_and_init('gpu', mx_lib) + utils.docker_run('ubuntu_nightly_gpu', 'nightly_estimator', true) + } + } + } } } , From 96bed657e317b3c83f4bc8ffb4798ed1c0e5c4bc Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Thu, 24 Oct 2019 13:57:15 -0700 Subject: [PATCH 6/9] add mkl lib --- tests/nightly/JenkinsfileForBinaries | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index a66159d0075b..44bf24798ebf 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -20,7 +20,7 @@ mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' -mx_lib_cpp_example = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, build/cpp-package/example/imagenet_inference' +mx_lib_cpp_example_mkl = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, build/cpp-package/example/imagenet_inference, lib/libmkldnn.so.0, lib/libmklml_intel.so' node('utility') { // Loading the utilities requires a node context unfortunately @@ -37,7 +37,7 @@ core_logic: { ws('workspace/build-gpu') { utils.init_git() utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda101_cudnn7', false) - utils.pack_lib('gpu', mx_lib_cpp_example) + utils.pack_lib('gpu', mx_lib_cpp_example_mkl) } } }/*, @@ -73,7 +73,7 @@ core_logic: { 'ImageNet Inference: GPU': { node(NODE_LINUX_GPU) { ws('workspace/nt-ImageInferenceTest') { - utils.unpack_and_init('gpu', mx_lib_cpp_example) + utils.unpack_and_init('gpu', mx_lib_cpp_example_mkl) utils.docker_run('ubuntu_nightly_gpu', 'nightly_test_imagenet_inference', true) } } From ef94c5d022c2c692b5368a60c14747e346f38571 Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Thu, 24 Oct 2019 14:29:55 -0700 Subject: [PATCH 7/9] update docker run build function --- tests/nightly/JenkinsfileForBinaries | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index 44bf24798ebf..af87b2c35658 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -34,9 +34,9 @@ core_logic: { stage('Build') { parallel 'GPU: CUDA10.1+cuDNN7': { node(NODE_LINUX_CPU) { - ws('workspace/build-gpu') { + ws('workspace/build-mkldnn-gpu') { utils.init_git() - utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda101_cudnn7', false) + utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', false) utils.pack_lib('gpu', mx_lib_cpp_example_mkl) } } From 98f8eef2057351d7964f1e9326ea6772c216f0af Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Thu, 24 Oct 2019 14:34:21 -0700 Subject: [PATCH 8/9] set nvidia docker true to run imagenet inference on GPU --- tests/nightly/JenkinsfileForBinaries | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index af87b2c35658..5ee14497123c 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -36,7 +36,7 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/build-mkldnn-gpu') { utils.init_git() - utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', false) + utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', true) utils.pack_lib('gpu', mx_lib_cpp_example_mkl) } } From 2b0ff215f8cb45d5c082a4230e2702bea4969ccf Mon Sep 17 00:00:00 2001 From: ChaiBapchya Date: Thu, 24 Oct 2019 15:20:09 -0700 Subject: [PATCH 9/9] Revert "set nvidia docker true to run imagenet inference on GPU" This reverts commit 98f8eef2057351d7964f1e9326ea6772c216f0af. As we don't need GPU for compilation. --- tests/nightly/JenkinsfileForBinaries | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index 5ee14497123c..af87b2c35658 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -36,7 +36,7 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/build-mkldnn-gpu') { utils.init_git() - utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', true) + utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', false) utils.pack_lib('gpu', mx_lib_cpp_example_mkl) } }