Support mkldnn for Scala.

apache · Jan 8, 2019 · 2f650a6 · 2f650a6
1 parent 9723454
commit 2f650a6
Show file tree

Hide file tree

Showing 6 changed files with 29 additions and 67 deletions.
diff --git a/Makefile b/Makefile
@@ -614,78 +614,16 @@ scalaclean:
 	(cd $(ROOTDIR)/scala-package && mvn clean)
 
 scalapkg:
-<<<<<<< HEAD
-	(cd $(ROOTDIR)/scala-package && \
-		mvn package $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
-		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dcurrent_libdir="$(ROOTDIR)/lib" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
-
-scalaunittest:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))
-
-scalaintegrationtest:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))
-
-scalainstall:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn install $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \
-		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
-
-scalarelease-dryrun:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \
-		-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
-		-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")
-
-scalarelease-prepare:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn release:clean release:prepare -DautoVersionSubmodules=true \
-		-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
-		-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")
-
-scalarelease-perform:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn release:perform -DautoVersionSubmodules=true \
-		-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
-		-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")
-
-scaladeploy:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn deploy $(MAVEN_ARGS) -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
-		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
-
-scaladeploylocal:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn deploy $(MAVEN_ARGS) -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
-		  -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \
-		  -Dgpg.skip \
-		  -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
-=======
 	(cd $(ROOTDIR)/scala-package && mvn install -DskipTests)
 
 scalainstall:
 	(cd $(ROOTDIR)/scala-package && mvn install)
 
 scalaunittest:
-	(cd $(ROOTDIR)/scala-package && mvn package)
+	(cd $(ROOTDIR)/scala-package && mvn install)
 
 scalaintegrationtest:
-	(cd $(ROOTDIR)/scala-package && mvn integration-test)
->>>>>>> Redo maven deploy related tasks.
+	(cd $(ROOTDIR)/scala-package && mvn integration-test -DskipTests=false)
 
 jnilint:
 	3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src --exclude_path scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h

diff --git a/scala-package/README.md b/scala-package/README.md
@@ -135,7 +135,7 @@ Or run a subset of unit tests, for e.g.,
 
 ```bash
 cd scala-package
-mvn -DSCALA_TEST_ARGS=-Dsuites=org.apache.mxnet.NDArraySuite integration-test
+mvn -Dsuites=org.apache.mxnet.NDArraySuite integration-test
 ```
 
 If everything goes well, you will find jars for `assembly`, `core` and `example` modules.
@@ -196,3 +196,7 @@ More details about JVM Memory Management are available [here](https://github.com
 License
 -------
 MXNet Scala Package is licensed under [Apache-2](https://github.com/apache/incubator-mxnet/blob/master/scala-package/LICENSE) license.
+
+MXNet uses some 3rd party softwares. Following 3rd party license files are bundled inside Scala jar file:
+* cub/LICENSE.TXT
+* mkldnn/external/mklml_mac_2019.0.1.20180928/license.txt
diff --git a/scala-package/assembly/src/main/assembly/assembly.xml b/scala-package/assembly/src/main/assembly/assembly.xml
@@ -36,8 +36,22 @@
         <include>libmxnet.so</include>
         <include>libgfortran.so.3</include>
         <include>libquadmath.so.0</include>
+        <include>libiomp5.so</include>
+        <include>libiomp5.dylib</include>
+        <include>libmklml_intel.so</include>
+        <include>libmklml.dylib</include>
+        <include>libmkldnn.so.0</include>
+        <include>libmkldnn.0.dylib</include>
       </includes>
       <outputDirectory>lib/native</outputDirectory>
     </fileSet>
+    <fileSet>
+      <directory>${MXNET_DIR}/3rdparty</directory>
+      <includes>
+        <include>cub/LICENSE.TXT</include>
+        <include>mkldnn/external/mklml_mac_2019.0.1.20180928/license.txt</include>
+      </includes>
+      <outputDirectory>/</outputDirectory>
+    </fileSet>
   </fileSets>
 </assembly>
diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala
@@ -88,6 +88,12 @@ private[mxnet] object NativeLibraryLoader {
     saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so", true)
     saveLibraryToTemp("libgfortran.so.3", "/lib/native/libgfortran.so.3", false)
     saveLibraryToTemp("libquadmath.so.0", "/lib/native/libquadmath.so.0", false)
+    saveLibraryToTemp("libiomp5.so", "/lib/native/libiomp5.so", false)
+    saveLibraryToTemp("libiomp5.dylib", "/lib/native/libiomp5.dylib", false)
+    saveLibraryToTemp("libmklml_intel.so", "/lib/native/libmklml_intel.so", false)
+    saveLibraryToTemp("libmklml.dylib", "/lib/native/libmklml.dylib", false)
+    saveLibraryToTemp("libmkldnn.so.0", "/lib/native/libmkldnn.so.0", false)
+    saveLibraryToTemp("libmkldnn.0.dylib", "/lib/native/libmkldnn.0.dylib", false)
     val tempfile: File = saveLibraryToTemp(libname, libFileInJar, true)
 
     loadLibraryFromFile(libname, tempfile)

diff --git a/scala-package/init-native/pom.xml b/scala-package/init-native/pom.xml
@@ -164,7 +164,7 @@
             </goals>
             <configuration>
               <executable>bash</executable>
-              <commandlineArgs>-c 'ln -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so; ln -sf ${MXNET_DIR}/lib/libgfortran.so.3 ${project.build.directory}/libgfortran.so.3; ln -sf ${MXNET_DIR}/lib/libquadmath.so.0 ${project.build.directory}/libquadmath.so.0'</commandlineArgs>
+              <commandlineArgs>-c 'ln -sf ${MXNET_DIR}/lib/* ${project.build.directory}/'</commandlineArgs>
             </configuration>
           </execution>
         </executions>

diff --git a/scala-package/native/pom.xml b/scala-package/native/pom.xml
@@ -159,7 +159,7 @@
             </goals>
             <configuration>
               <executable>bash</executable>
-              <commandlineArgs>-c 'ln -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so; ln -sf ${MXNET_DIR}/lib/libgfortran.so.3 ${project.build.directory}/libgfortran.so.3; ln -sf ${MXNET_DIR}/lib/libquadmath.so.0 ${project.build.directory}/libquadmath.so.0'</commandlineArgs>
+              <commandlineArgs>-c 'ln -sf ${MXNET_DIR}/lib/* ${project.build.directory}/'</commandlineArgs>
             </configuration>
           </execution>
         </executions>