Merge remote-tracking branch 'upstream/master'

* upstream/master: (38 commits) Feature/mkldnn static (apache#13628) Fix the bug of BidirectionalCell (apache#13575) Set install path for libmxnet.so dynamic lib on Mac OS (apache#13629) add batch norm test (apache#13625) Scripts for building dependency libraries of MXNet (apache#13282) fix quantize pass error when the quantization supported Op are excluded in the model (apache#13596) Optimize C++ API (apache#13496) Fix warning in waitall doc (apache#13618) [MXNET-1225] Always use config.mk in make install instructions (apache#13364) [MXNET-1224]: improve scala maven jni build and packing. (apache#13493) [MXNET-1155] Add scala packageTest utility (apache#13046) fix the Float not showing correctly problem (apache#13617) apache#13385 [Clojure] - Turn examples into integration tests (apache#13554) Add Intel MKL blas to Jenkins (apache#13607) Revert "[MXNET-1198] MXNet Java API (apache#13162)" Reducing the length of setup tutorial (apache#13306) [MXNET-1182] Predictor example (apache#13237) [MXNET-1187] Added Java SSD Inference Tutorial for website (apache#13201) add defaults and clean up the tests (apache#13295) [MXNET-1181] Added command line alternative to IntelliJ in install instructions (apache#13267) ...
zhaoyao73 · Dec 13, 2018 · 76e901a · 76e901a
2 parents 753cfb0 + 5bcf2bd
commit 76e901a
Show file tree

Hide file tree

Showing 94 changed files with 2,810 additions and 301 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -227,6 +227,7 @@ if(USE_MKLDNN)
   include(cmake/DownloadMKLML.cmake)
   # CPU architecture (e.g., C5) can't run on another architecture (e.g., g3).
   if(NOT MSVC)
+    set(MKLDNN_LIBRARY_TYPE "STATIC" CACHE INTERNAL "" FORCE)
     set(ARCH_OPT_FLAGS "-mtune=generic")
   else()
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /EHsc")

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
@@ -192,6 +192,7 @@ List of Contributors
 * [Rahul Padmanabhan](https://github.com/rahul3)
 * [Yuxi Hu](https://github.com/yuxihu)
 * [Harsh Patel](https://github.com/harshp8l)
+* [Xiao Wang](https://github.com/BeyonderXX)
 
 Label Bot
 ---------

diff --git a/Makefile b/Makefile
@@ -132,7 +132,12 @@ ifeq ($(USE_MKLDNN), 1)
 		LDFLAGS += -L$(MKLROOT)/lib
 	endif
 	CFLAGS += -I$(MKLDNNROOT)/include
-	LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
+	# MKLDNN but to needs to be dynamically linked for windows as not all VS compilers support static linking
+	ifneq ($(UNAME_S), Windows)
+		LIB_DEP += $(MKLDNNROOT)/lib/libmkldnn.a
+	else
+		LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
+	endif
 endif
 
 # setup opencv
@@ -486,6 +491,11 @@ build/plugin/%.o: plugin/%.cc
 	@mkdir -p $(@D)
 	$(CXX) -std=c++11 -c $(CFLAGS) -MMD -Isrc/operator -c $< -o $@
 
+# Set install path for libmxnet.so on Mac OS
+ifeq ($(UNAME_S), Darwin)
+        LDFLAGS += -Wl,-install_name,@rpath/libmxnet.so
+endif
+
 # NOTE: to statically link libmxnet.a we need the option
 # --Wl,--whole-archive -lmxnet --Wl,--no-whole-archive
 lib/libmxnet.a: $(ALLX_DEP)
@@ -600,61 +610,78 @@ rpkgtest:
 	Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file(paste("r-package_coverage_",toString(runif(1)),".json"));writeLines(covr:::to_codecov(res), fileConn);close(fileConn)'
 
 scalaclean:
-	(cd $(ROOTDIR)/scala-package; \
+	(cd $(ROOTDIR)/scala-package && \
 		mvn clean -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE))
 
+scalatestcompile:
+	(cd $(ROOTDIR)/scala-package && \
+		mvn test-compile -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
+		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
+			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
+			-Dcurrent_libdir="$(ROOTDIR)/lib" \
+			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
+
 scalapkg:
-	(cd $(ROOTDIR)/scala-package; \
+	(cd $(ROOTDIR)/scala-package && \
 		mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
 		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
 			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
 			-Dcurrent_libdir="$(ROOTDIR)/lib" \
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
 
 scalaunittest:
-	(cd $(ROOTDIR)/scala-package; \
+	(cd $(ROOTDIR)/scala-package && \
 		mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \
 			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))
 
 scalaintegrationtest:
-	(cd $(ROOTDIR)/scala-package; \
+	(cd $(ROOTDIR)/scala-package && \
 		mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \
 			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))
 
 scalainstall:
-	(cd $(ROOTDIR)/scala-package; \
+	(cd $(ROOTDIR)/scala-package && \
 		mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \
 		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
 			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
 
 scalarelease-dryrun:
-	(cd $(ROOTDIR)/scala-package; \
+	(cd $(ROOTDIR)/scala-package && \
 		mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \
 		-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
 		-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")
 
 scalarelease-prepare:
-	(cd $(ROOTDIR)/scala-package; \
+	(cd $(ROOTDIR)/scala-package && \
 		mvn release:clean release:prepare -DautoVersionSubmodules=true \
 		-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
 		-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")
 
 scalarelease-perform:
-	(cd $(ROOTDIR)/scala-package; \
+	(cd $(ROOTDIR)/scala-package && \
 		mvn release:perform -DautoVersionSubmodules=true \
 		-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
 		-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")
 
 scaladeploy:
-	(cd $(ROOTDIR)/scala-package; \
+	(cd $(ROOTDIR)/scala-package && \
 		mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
 		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
 			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
 
+scaladeploylocal:
+	(cd $(ROOTDIR)/scala-package && \
+		mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
+		  -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \
+		  -Dgpg.skip \
+		  -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
+			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
+			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
+
 jnilint:
 	3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -54,6 +54,9 @@ RUN /work/ubuntu_clang.sh
 COPY install/ubuntu_gcc8.sh /work/
 RUN /work/ubuntu_gcc8.sh
 
+COPY install/ubuntu_mkl.sh /work/
+RUN /work/ubuntu_mkl.sh
+
 COPY install/ubuntu_mklml.sh /work/
 RUN /work/ubuntu_mklml.sh
 

diff --git a/ci/docker/install/ubuntu_mkl.sh b/ci/docker/install/ubuntu_mkl.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+
+apt-get update || true
+# Install Intel Math Kernel Library (latest major release)
+# https://software.intel.com/en-us/articles/installing-intel-free-libs-and-python-apt-repo
+wget -O - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB | apt-key add - && \
+    sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \
+    apt-get update && \
+    apt-get install -y intel-mkl-2019.1-053
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
@@ -326,6 +326,20 @@ build_ubuntu_cpu_openblas() {
         -j$(nproc)
 }
 
+build_ubuntu_cpu_mkl() {
+    set -ex
+    export CC="ccache gcc"
+    export CXX="ccache g++"
+    make \
+        DEV=1                         \
+        ENABLE_TESTCOVERAGE=1         \
+        USE_CPP_PACKAGE=1             \
+        USE_BLAS=mkl                  \
+        USE_INTEL_PATH=/opt/intel     \
+        USE_DIST_KVSTORE=1            \
+        -j$(nproc)
+}
+
 build_ubuntu_cpu_cmake_debug() {
     set -ex
     pushd .
@@ -480,6 +494,20 @@ build_ubuntu_cpu_mkldnn() {
         -j$(nproc)
 }
 
+build_ubuntu_cpu_mkldnn_mkl() {
+    set -ex
+
+    build_ccache_wrappers
+
+    make  \
+        DEV=1                         \
+        ENABLE_TESTCOVERAGE=1         \
+        USE_CPP_PACKAGE=1             \
+        USE_BLAS=mkl                  \
+        USE_MKLDNN=1                  \
+        -j$(nproc)
+}
+
 build_ubuntu_gpu() {
     build_ubuntu_gpu_cuda91_cudnn7
 }
@@ -629,9 +657,6 @@ build_ubuntu_gpu_cmake_mkldnn() {
         /work/mxnet
 
     ninja -v
-    # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0.
-    cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
-    mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0
 }
 
 build_ubuntu_gpu_cmake() {

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
@@ -23,19 +23,19 @@
 utils = load('ci/Jenkinsfile_utils.groovy')
 
 // mxnet libraries
-mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 
 // Python wheels
 mx_pip = 'build/*.whl'
 
 // for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
+mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmkldnn.a'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
-mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
-mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
+mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
 mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*'
 mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*'
@@ -121,6 +121,20 @@ def compile_unix_openblas_debug_cpu() {
     }]
 }
 
+def compile_unix_mkl_cpu() {
+    return ['CPU: MKL': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/build-cpu-mkl') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.init_git()
+            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkl', false)
+            utils.pack_lib('cpu_mkl', mx_dist_lib, true)
+          }
+        }
+      }
+    }]
+}
+
 def compile_unix_mkldnn_cpu() {
     return ['CPU: MKLDNN': {
       node(NODE_LINUX_CPU) {
@@ -135,6 +149,20 @@ def compile_unix_mkldnn_cpu() {
     }]
 }
 
+def compile_unix_mkldnn_mkl_cpu() {
+    return ['CPU: MKLDNN_MKL': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/build-mkldnn-cpu') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.init_git()
+            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn_mkl', false)
+            utils.pack_lib('mkldnn_mkl_cpu', mx_mkldnn_lib, true)
+          }
+        }
+      }
+    }]
+}
+
 def compile_unix_mkldnn_gpu() {
     return ['GPU: MKLDNN': {
       node(NODE_LINUX_CPU) {
@@ -580,6 +608,23 @@ def test_unix_python3_cpu() {
     }]
 }
 
+def test_unix_python3_mkl_cpu() {
+    return ['Python3: MKL-CPU': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/ut-python3-cpu') {
+          try {
+            utils.unpack_and_init('cpu_mkl', mx_lib, true)
+            python3_ut('ubuntu_cpu')
+            utils.publish_test_coverage()
+          } finally {
+            utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_unittest.xml')
+            utils.collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python3_cpu_quantization.xml')
+          }
+        }
+      }
+    }]
+}
+
 def test_unix_python3_gpu() {
     return ['Python3: GPU': {
       node(NODE_LINUX_GPU) {
@@ -665,6 +710,23 @@ def test_unix_python3_mkldnn_cpu() {
     }]
 }
 
+def test_unix_python3_mkldnn_mkl_cpu() {
+    return ['Python3: MKLDNN-MKL-CPU': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/ut-python3-mkldnn-mkl-cpu') {
+          try {
+            utils.unpack_and_init('mkldnn_mkl_cpu', mx_mkldnn_lib, true)
+            python3_ut_mkldnn('ubuntu_cpu')
+            utils.publish_test_coverage()
+          } finally {
+            utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_mkldnn_cpu_unittest.xml')
+            utils.collect_test_results_unix('nosetests_mkl.xml', 'nosetests_python3_mkldnn_cpu_mkl.xml')
+          }
+        }
+      }
+    }]
+}
+
 def test_unix_python3_mkldnn_gpu() {
     return ['Python3: MKLDNN-GPU': {
       node(NODE_LINUX_GPU) {

diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu
@@ -36,15 +36,19 @@ core_logic: {
   utils.parallel_stage('Build', [
     custom_steps.compile_unix_cpu_openblas(),
     custom_steps.compile_unix_openblas_debug_cpu(),
-    custom_steps.compile_unix_mkldnn_cpu()
+    custom_steps.compile_unix_mkl_cpu(),
+    custom_steps.compile_unix_mkldnn_cpu(),
+    custom_steps.compile_unix_mkldnn_mkl_cpu()
   ]) 
 
   utils.parallel_stage('Tests', [
     custom_steps.test_unix_python2_cpu(),
     custom_steps.test_unix_python3_cpu(),
     custom_steps.test_unix_python3_debug_cpu(),
+    custom_steps.test_unix_python3_mkl_cpu(),
     custom_steps.test_unix_python2_mkldnn_cpu(),
     custom_steps.test_unix_python3_mkldnn_cpu(),
+    custom_steps.test_unix_python3_mkldnn_mkl_cpu(),
     custom_steps.test_unix_scala_cpu(),
     custom_steps.test_unix_clojure_cpu(),
     custom_steps.test_unix_r_cpu(),