From 8608daad56cc8b8d277cf60d32308038b911d8ce Mon Sep 17 00:00:00 2001 From: Anton Chernov Date: Wed, 12 Sep 2018 17:08:05 +0200 Subject: [PATCH] Used jom for parallel windows builds --- Jenkinsfile | 811 +------------------------------------------- MKLDNN_README.md | 4 +- appveyor.yml | 2 +- ci/build_windows.py | 3 +- 4 files changed, 7 insertions(+), 813 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 346cb19ce46f..2fab84982649 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -88,242 +88,9 @@ utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu utils.main_wrapper( core_logic: { - stage('Sanity Check') { - parallel 'Lint': { - node(NODE_LINUX_CPU) { - ws('workspace/sanity-lint') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'sanity_check', false) - } - } - }, - 'RAT License': { - node(NODE_LINUX_CPU) { - ws('workspace/sanity-rat') { - utils.init_git() - utils.docker_run('ubuntu_rat', 'nightly_test_rat_check', false) - } - } - } - } stage('Build') { - parallel 'CPU: CentOS 7': { - node(NODE_LINUX_CPU) { - ws('workspace/build-centos7-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('centos7_cpu', 'build_centos7_cpu', false) - utils.pack_lib('centos7_cpu', mx_lib) - } - } - } - }, - 'CPU: CentOS 7 MKLDNN': { - node(NODE_LINUX_CPU) { - ws('workspace/build-centos7-mkldnn') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('centos7_cpu', 'build_centos7_mkldnn', false) - utils.pack_lib('centos7_mkldnn', mx_lib) - } - } - } - }, - 'GPU: CentOS 7': { - node(NODE_LINUX_CPU) { - ws('workspace/build-centos7-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('centos7_gpu', 'build_centos7_gpu', false) - utils.pack_lib('centos7_gpu', mx_lib) - } - } - } - }, - 'CPU: Openblas': { - node(NODE_LINUX_CPU) { - ws('workspace/build-cpu-openblas') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false) - utils.pack_lib('cpu', mx_dist_lib) - } - } - } - }, - 'CPU: Openblas, debug': { - node(NODE_LINUX_CPU) { - ws('workspace/build-cpu-openblas') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_cmake_debug', false) - utils.pack_lib('cpu_debug', mx_cmake_lib_debug) - } - } - } - }, - 'CPU: Clang 3.9': { - node(NODE_LINUX_CPU) { - ws('workspace/build-cpu-clang39') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39', false) - } - } - } - }, - 'CPU: Clang 6': { - node(NODE_LINUX_CPU) { - ws('workspace/build-cpu-clang60') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang60', false) - } - } - } - }, - 'CPU: Clang Tidy': { - node(NODE_LINUX_CPU) { - ws('workspace/build-cpu-clang60_tidy') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang_tidy', false) - } - } - } - }, - 'CPU: Clang 3.9 MKLDNN': { - node(NODE_LINUX_CPU) { - ws('workspace/build-cpu-mkldnn-clang39') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39_mkldnn', false) - utils.pack_lib('mkldnn_cpu_clang3', mx_mkldnn_lib) - } - } - } - }, - 'CPU: Clang 6 MKLDNN': { - node(NODE_LINUX_CPU) { - ws('workspace/build-cpu-mkldnn-clang60') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang60_mkldnn', false) - utils.pack_lib('mkldnn_cpu_clang6', mx_mkldnn_lib) - } - } - } - }, - 'CPU: MKLDNN': { - node(NODE_LINUX_CPU) { - ws('workspace/build-mkldnn-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn', false) - utils.pack_lib('mkldnn_cpu', mx_mkldnn_lib) - } - } - } - }, - 'GPU: MKLDNN': { - node(NODE_LINUX_CPU) { - ws('workspace/build-mkldnn-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', false) - utils.pack_lib('mkldnn_gpu', mx_mkldnn_lib) - } - } - } - }, - 'GPU: MKLDNN_CUDNNOFF': { - node(NODE_LINUX_CPU) { - ws('workspace/build-mkldnn-gpu-nocudnn') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn_nocudnn', false) - utils.pack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) - } - } - } - }, - 'GPU: CUDA9.1+cuDNN7': { - node(NODE_LINUX_CPU) { - ws('workspace/build-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7', false) - utils.pack_lib('gpu', mx_dist_lib) - stash includes: 'build/cpp-package/example/lenet', name: 'cpp_lenet' - stash includes: 'build/cpp-package/example/alexnet', name: 'cpp_alexnet' - stash includes: 'build/cpp-package/example/googlenet', name: 'cpp_googlenet' - stash includes: 'build/cpp-package/example/lenet_with_mxdataiter', name: 'cpp_lenet_with_mxdataiter' - stash includes: 'build/cpp-package/example/resnet', name: 'cpp_resnet' - stash includes: 'build/cpp-package/example/mlp', name: 'cpp_mlp' - stash includes: 'build/cpp-package/example/mlp_cpu', name: 'cpp_mlp_cpu' - stash includes: 'build/cpp-package/example/mlp_gpu', name: 'cpp_mlp_gpu' - stash includes: 'build/cpp-package/example/test_score', name: 'cpp_test_score' - stash includes: 'build/cpp-package/example/test_optimizer', name: 'cpp_test_optimizer' - } - } - } - }, - 'Amalgamation MIN': { - node(NODE_LINUX_CPU) { - ws('workspace/amalgamationmin') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation_min', false) - } - } - } - }, - 'Amalgamation': { - node(NODE_LINUX_CPU) { - ws('workspace/amalgamation') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation', false) - } - } - } - }, - - 'GPU: CMake MKLDNN': { - node(NODE_LINUX_CPU) { - ws('workspace/build-cmake-mkldnn-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake_mkldnn', false) - utils.pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib) - } - } - } - }, - 'GPU: CMake': { - node(NODE_LINUX_CPU) { - ws('workspace/build-cmake-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false) - utils.pack_lib('cmake_gpu', mx_cmake_lib) - } - } - } - }, - 'TensorRT': { - node(NODE_LINUX_CPU) { - ws('workspace/build-tensorrt') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_gpu_tensorrt', 'build_ubuntu_gpu_tensorrt', false) - utils.pack_lib('tensorrt', mx_tensorrt_lib) - } - } - } - }, - 'Build CPU windows':{ + parallel 'Build CPU windows':{ node(NODE_WINDOWS_CPU) { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-cpu') { @@ -350,6 +117,7 @@ core_logic: { } } }, + 'Build GPU MKLDNN windows':{ node(NODE_WINDOWS_CPU) { timeout(time: max_time, unit: 'MINUTES') { @@ -362,585 +130,10 @@ core_logic: { } } } - }, - 'NVidia Jetson / ARMv8':{ - node(NODE_LINUX_CPU) { - ws('workspace/build-jetson-armv8') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('jetson', 'build_jetson', false) - } - } - } - }, - 'ARMv7':{ - node(NODE_LINUX_CPU) { - ws('workspace/build-ARMv7') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('armv7', 'build_armv7', false) - } - } - } - }, - 'ARMv6':{ - node(NODE_LINUX_CPU) { - ws('workspace/build-ARMv6') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('armv6', 'build_armv6', false) - } - } - } - }, - 'ARMv8':{ - node(NODE_LINUX_CPU) { - ws('workspace/build-ARMv8') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('armv8', 'build_armv8', false) - } - } - } - }, - 'Android / ARMv8':{ - node(NODE_LINUX_CPU) { - ws('workspace/android64') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('android_armv8', 'build_android_armv8', false) - } - } - } - }, - 'Android / ARMv7':{ - node(NODE_LINUX_CPU) { - ws('workspace/androidv7') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('android_armv7', 'build_android_armv7', false) - } - } - } } } // End of stage('Build') - stage('Tests') { - parallel 'Python2: CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/ut-python2-cpu') { - try { - utils.init_git() - utils.unpack_lib('cpu', mx_lib) - python2_ut('ubuntu_cpu') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_cpu_unittest.xml') - utils.collect_test_results_unix('nosetests_train.xml', 'nosetests_python2_cpu_train.xml') - utils.collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python2_cpu_quantization.xml') - } - } - } - }, - 'Python3: CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/ut-python3-cpu') { - try { - utils.init_git() - utils.unpack_lib('cpu', mx_lib) - python3_ut('ubuntu_cpu') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_unittest.xml') - utils.collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python3_cpu_quantization.xml') - } - } - } - }, - 'Python3: CPU debug': { - node(NODE_LINUX_CPU) { - ws('workspace/ut-python3-cpu-debug') { - try { - utils.init_git() - utils.unpack_lib('cpu_debug', mx_cmake_lib_debug) - python3_ut('ubuntu_cpu') - } finally { - utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_debug_unittest.xml') - utils.collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python3_cpu_debug_quantization.xml') - } - } - } - }, - 'Python2: GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-python2-gpu') { - try { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) - python2_gpu_ut('ubuntu_gpu') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_gpu.xml') - } - } - } - }, - 'Python3: GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-python3-gpu') { - try { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) - python3_gpu_ut('ubuntu_gpu') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml') - } - } - } - }, - 'Python2: Quantize GPU': { - node(NODE_LINUX_GPU_P3) { - ws('workspace/ut-python2-quantize-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - try { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) - utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_python2_quantization_gpu', true) - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python2_quantize_gpu.xml') - } - } - } - } - }, - 'Python3: Quantize GPU': { - node(NODE_LINUX_GPU_P3) { - ws('workspace/ut-python3-quantize-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - try { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) - utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_python3_quantization_gpu', true) - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python3_quantize_gpu.xml') - } - } - } - } - }, - 'Python2: MKLDNN-CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/ut-python2-mkldnn-cpu') { - try { - utils.init_git() - utils.unpack_lib('mkldnn_cpu', mx_mkldnn_lib) - python2_ut('ubuntu_cpu') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_mkldnn_cpu_unittest.xml') - utils.collect_test_results_unix('nosetests_train.xml', 'nosetests_python2_mkldnn_cpu_train.xml') - utils.collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python2_mkldnn_cpu_quantization.xml') - } - } - } - }, - 'Python2: MKLDNN-GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-python2-mkldnn-gpu') { - try { - utils.init_git() - utils.unpack_lib('mkldnn_gpu', mx_mkldnn_lib) - python2_gpu_ut('ubuntu_gpu') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_mkldnn_gpu.xml') - } - } - } - }, - 'Python3: MKLDNN-CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/ut-python3-mkldnn-cpu') { - try { - utils.init_git() - utils.unpack_lib('mkldnn_cpu', mx_mkldnn_lib) - python3_ut_mkldnn('ubuntu_cpu') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_mkldnn_cpu_unittest.xml') - utils.collect_test_results_unix('nosetests_mkl.xml', 'nosetests_python3_mkldnn_cpu_mkl.xml') - } - } - } - }, - 'Python3: MKLDNN-GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-python3-mkldnn-gpu') { - try { - utils.init_git() - utils.unpack_lib('mkldnn_gpu', mx_mkldnn_lib) - python3_gpu_ut('ubuntu_gpu') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu.xml') - } - } - } - }, - 'Python3: MKLDNN-GPU-NOCUDNN': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-python3-mkldnn-gpu-nocudnn') { - try { - utils.init_git() - utils.unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) - python3_gpu_ut_nocudnn('ubuntu_gpu') - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu_nocudnn.xml') - } - } - } - }, - 'Python3: CentOS 7 CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/build-centos7-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - try { - utils.init_git() - utils.unpack_lib('centos7_cpu', mx_lib) - utils.docker_run('centos7_cpu', 'unittest_centos7_cpu', false) - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_centos7_cpu_unittest.xml') - utils.collect_test_results_unix('nosetests_train.xml', 'nosetests_python3_centos7_cpu_train.xml') - } - } - } - } - }, - 'Python3: CentOS 7 GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/build-centos7-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - try { - utils.init_git() - utils.unpack_lib('centos7_gpu', mx_lib) - utils.docker_run('centos7_gpu', 'unittest_centos7_gpu', true) - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_centos7_gpu.xml') - } - } - } - } - }, - 'Python3: TensorRT GPU': { - node(NODE_LINUX_GPU_P3) { - ws('workspace/build-tensorrt') { - timeout(time: max_time, unit: 'MINUTES') { - try { - utils.init_git() - utils.unpack_lib('tensorrt', mx_tensorrt_lib) - utils.docker_run('ubuntu_gpu_tensorrt', 'unittest_ubuntu_tensorrt_gpu', true) - utils.publish_test_coverage() - } finally { - utils.collect_test_results_unix('nosetests_tensorrt.xml', 'nosetests_python3_tensorrt_gpu.xml') - } - } - } - } - }, - 'Scala: CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/ut-scala-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cpu', mx_dist_lib) - utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false) - utils.publish_test_coverage() - } - } - } - }, - 'Clojure: CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/ut-clojure-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cpu', mx_dist_lib) - utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_clojure', false) - utils.publish_test_coverage() - } - } - } - }, - 'Perl: CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/ut-perl-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpugpu_perl', false) - utils.publish_test_coverage() - } - } - } - }, - 'Perl: GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-perl-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) - utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true) - utils.publish_test_coverage() - } - } - } - }, - 'Cpp: GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-cpp-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cmake_gpu', mx_cmake_lib) - utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true) - utils.publish_test_coverage() - } - } - } - }, - 'Cpp: MKLDNN+GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-cpp-mkldnn-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib) - utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true) - utils.publish_test_coverage() - } - } - } - }, - 'R: CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/ut-r-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_R', false) - utils.publish_test_coverage() - } - } - } - }, - 'R: GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-r-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) - utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true) - utils.publish_test_coverage() - } - } - } - }, - - 'Python 2: CPU Win':{ - node(NODE_WINDOWS_CPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/ut-python-cpu') { - try { - utils.init_git_win() - unstash 'windows_package_cpu' - powershell 'ci/windows/test_py2_cpu.ps1' - } finally { - utils.collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python2_cpu.xml') - } - } - } - } - }, - 'Python 3: CPU Win': { - node(NODE_WINDOWS_CPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/ut-python-cpu') { - try { - utils.init_git_win() - unstash 'windows_package_cpu' - powershell 'ci/windows/test_py3_cpu.ps1' - } finally { - utils.collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python3_cpu.xml') - } - } - } - } - }, - 'Python 2: GPU Win':{ - node(NODE_WINDOWS_GPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/ut-python-gpu') { - try { - utils.init_git_win() - unstash 'windows_package_gpu' - powershell 'ci/windows/test_py2_gpu.ps1' - } finally { - utils.collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml') - utils.collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml') - } - } - } - } - }, - 'Python 3: GPU Win':{ - node(NODE_WINDOWS_GPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/ut-python-gpu') { - try { - utils.init_git_win() - unstash 'windows_package_gpu' - powershell 'ci/windows/test_py3_gpu.ps1' - } finally { - utils.collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml') - utils.collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml') - } - } - } - } - }, - 'Python 3: MKLDNN-GPU Win':{ - node(NODE_WINDOWS_GPU) { - timeout(time: max_time, unit: 'MINUTES') { - ws('workspace/ut-python-gpu') { - try { - utils.init_git_win() - unstash 'windows_package_gpu_mkldnn' - powershell 'ci/windows/test_py3_gpu.ps1' - } finally { - utils.collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml') - utils.collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml') - } - } - } - } - }, - 'Onnx CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/it-onnx-cpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false) - utils.publish_test_coverage() - } - } - } - }, - 'Python GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/it-python-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) - utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true) - utils.publish_test_coverage() - } - } - } - }, - // Disabled due to: https://github.com/apache/incubator-mxnet/issues/11407 - // 'Caffe GPU': { - // node(NODE_LINUX_GPU) { - // ws('workspace/it-caffe') { - // timeout(time: max_time, unit: 'MINUTES') { - // utils.init_git() - // utils.unpack_lib('gpu', mx_lib) - // utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_caffe', true) - // utils.publish_test_coverage() - // } - // } - // } - // }, - 'cpp-package GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/it-cpp-package') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) - unstash 'cpp_lenet' - unstash 'cpp_alexnet' - unstash 'cpp_googlenet' - unstash 'cpp_lenet_with_mxdataiter' - unstash 'cpp_resnet' - unstash 'cpp_mlp' - unstash 'cpp_mlp_cpu' - unstash 'cpp_mlp_gpu' - unstash 'cpp_test_score' - unstash 'cpp_test_optimizer' - utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_cpp_package', true) - utils.publish_test_coverage() - } - } - } - }, - 'dist-kvstore tests GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/it-dist-kvstore') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) - utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_dist_kvstore', true) - utils.publish_test_coverage() - } - } - } - }, - /* Disabled due to master build failure: - * http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/ - * https://github.com/apache/incubator-mxnet/issues/11801 - - 'dist-kvstore tests CPU': { - node(NODE_LINUX_CPU) { - ws('workspace/it-dist-kvstore') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cpu', mx_lib) - utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_dist_kvstore', false) - utils.publish_test_coverage() - } - } - } - }, */ - 'Scala: GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/ut-scala-gpu') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('gpu', mx_dist_lib) - utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true) - utils.publish_test_coverage() - } - } - } - } - } - - stage('Deploy') { - node(NODE_LINUX_CPU) { - ws('workspace/docs') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'deploy_docs', false) - sh "ci/other/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}" - } - } - } - } } , failure_handler: { diff --git a/MKLDNN_README.md b/MKLDNN_README.md index 43cced49ed0d..9ada62e452e9 100644 --- a/MKLDNN_README.md +++ b/MKLDNN_README.md @@ -126,7 +126,7 @@ git clone --recursive https://github.com/apache/incubator-mxnet.git ``` mkdir build cd build -cmake -G "Visual Studio 14 Win64" .. -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DUSE_MKLDNN=1 -DCMAKE_BUILD_TYPE=Release +cmake -G "Visual Studio 14 Win64" .. -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DUSE_MKLDNN=1 -DCMAKE_BUILD_TYPE=Release ``` 5. In Visual Studio, open the solution file,```.sln```, and compile it. @@ -178,7 +178,7 @@ cd C:\build 7. CMake the MXNet source code by using following command: ```r -cmake -G "Visual Studio 15 2017 Win64" .. -T host=x64 -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DUSE_MKLDNN=1 -DCMAKE_BUILD_TYPE=Release +cmake -G "Visual Studio 15 2017 Win64" .. -T host=x64 -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DUSE_MKLDNN=1 -DCMAKE_BUILD_TYPE=Release ``` 8. After the CMake successfully completed, compile the the MXNet source code by using following command: diff --git a/appveyor.yml b/appveyor.yml index d44f52a0a9a9..abd3858d077c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -69,7 +69,7 @@ before_build: set OpenCV_DIR=%APPVEYOR_BUILD_FOLDER%/%MXNET_OPENCV_DIR%/build - cmake .. -DOPENCV_DIR=%OpenCV_DIR% -DUSE_PROFILER=1 -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -G "Visual Studio 12 2013 Win64" + cmake .. -DOPENCV_DIR=%OpenCV_DIR% -DUSE_PROFILER=1 -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -G "Visual Studio 12 2013 Win64" build_script: - cmd: >- diff --git a/ci/build_windows.py b/ci/build_windows.py index 5eca58db7b74..87e01c5ec6db 100755 --- a/ci/build_windows.py +++ b/ci/build_windows.py @@ -149,7 +149,8 @@ def windows_build(args): check_call("cmake -G \"Visual Studio 14 2015 Win64\" {} {}".format(CMAKE_FLAGS[args.flavour], mxnet_root), shell=True) logging.info("Building with visual studio") t0 = int(time.time()) - check_call(["msbuild", "mxnet.sln","/p:configuration=release;platform=x64", "/maxcpucount","/v:minimal"]) + check_call(["msbuild", "mxnet.sln", "/p:configuration=release;platform=x64", "/maxcpucount", "/v:minimal", + "/p:BuildInParallel=true"]) logging.info("Build flavour: %s complete in directory: \"%s\"", args.flavour, os.path.abspath(path)) logging.info("Build took %s" , datetime.timedelta(seconds=int(time.time()-t0))) windows_package(args)