Redo maven deploy related tasks.

1. Removed maven release plugin. 2. Make maven build friendly to CI, allow cli override version. 3. Moved gpg signing to deploy stage. 4. Created a separeated deploy module. 5. Updated Makefile to new maven build change. 6. Remove unused nexus-staging-plugin 7. Added nightly and staging profile for CI.
apache · Jan 2, 2019 · 207fb3b · 207fb3b
1 parent abeec58
commit 207fb3b
Show file tree

Hide file tree

Showing 24 changed files with 394 additions and 451 deletions.
diff --git a/Makefile b/Makefile
@@ -18,8 +18,6 @@
 ROOTDIR = $(CURDIR)
 TPARTYDIR = $(ROOTDIR)/3rdparty
 
-SCALA_VERSION_PROFILE := scala-2.11
-
 ifeq ($(OS),Windows_NT)
 	UNAME_S := Windows
 else
@@ -397,18 +395,13 @@ PLUGIN_OBJ =
 PLUGIN_CUOBJ =
 include $(MXNET_PLUGINS)
 
-ifeq ($(UNAME_S), Windows)
-	# TODO(yizhi) currently scala package does not support windows
-	SCALA_PKG_PROFILE := windows
-else
+ifneq ($(UNAME_S), Windows)
 	ifeq ($(UNAME_S), Darwin)
 		WHOLE_ARCH= -all_load
 		NO_WHOLE_ARCH= -noall_load
-		SCALA_PKG_PROFILE := osx-x86_64
 	else
 		WHOLE_ARCH= --whole-archive
 		NO_WHOLE_ARCH= --no-whole-archive
-		SCALA_PKG_PROFILE := linux-x86_64
 	endif
 endif
 
@@ -427,7 +420,6 @@ ifeq ($(USE_CUDA), 1)
 	# Make sure to add stubs as fallback in order to be able to build
 	# without full CUDA install (especially if run without nvidia-docker)
 	LDFLAGS += -L/usr/local/cuda/lib64/stubs
-	SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-gpu
 	ifeq ($(USE_NCCL), 1)
 		ifneq ($(USE_NCCL_PATH), NONE)
 			CFLAGS += -I$(USE_NCCL_PATH)/include
@@ -439,7 +431,6 @@ ifeq ($(USE_CUDA), 1)
 		CFLAGS += -DMXNET_USE_NCCL=0
 	endif
 else
-	SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-cpu
 	CFLAGS += -DMXNET_USE_NCCL=0
 endif
 
@@ -605,77 +596,19 @@ rpkgtest:
 	Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file(paste("r-package_coverage_",toString(runif(1)),".json"));writeLines(covr:::to_codecov(res), fileConn);close(fileConn)'
 
 scalaclean:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn clean -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE))
-
-scalatestcompile:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn test-compile -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
-		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dcurrent_libdir="$(ROOTDIR)/lib" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
+	(cd $(ROOTDIR)/scala-package && mvn clean)
 
 scalapkg:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
-		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dcurrent_libdir="$(ROOTDIR)/lib" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
+	(cd $(ROOTDIR)/scala-package && mvn install -DskipTests)
+
+scalainstall:
+	(cd $(ROOTDIR)/scala-package && mvn install)
 
 scalaunittest:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))
+	(cd $(ROOTDIR)/scala-package && mvn package)
 
 scalaintegrationtest:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))
-
-scalainstall:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \
-		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
-
-scalarelease-dryrun:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \
-		-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
-		-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")
-
-scalarelease-prepare:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn release:clean release:prepare -DautoVersionSubmodules=true \
-		-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
-		-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")
-
-scalarelease-perform:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn release:perform -DautoVersionSubmodules=true \
-		-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
-		-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")
-
-scaladeploy:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
-		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
-
-scaladeploylocal:
-	(cd $(ROOTDIR)/scala-package && \
-		mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
-		  -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \
-		  -Dgpg.skip \
-		  -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
+	(cd $(ROOTDIR)/scala-package && mvn integration-test)
 
 jnilint:
 	3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src --exclude_path scala-package/native/src/main/native/org_apache_mxnet_native_c_api.h

diff --git a/contrib/clojure-package/project.clj b/contrib/clojure-package/project.clj
@@ -29,7 +29,7 @@
                  ;[org.apache.mxnet/mxnet-full_2.11-linux-x86_64-gpu "1.2.1"]
 
                  ;;; CI
-                 [org.apache.mxnet/mxnet-full_2.11 "1.5.0-SNAPSHOT"]
+                 [org.apache.mxnet/mxnet-full_2.11 "INTERNAL"]
 
                  [org.clojure/tools.logging "0.4.0"]
                  [org.apache.logging.log4j/log4j-core "2.8.1"]

diff --git a/docs/mxdoc.py b/docs/mxdoc.py
@@ -113,9 +113,9 @@ def build_scala_docs(app):
     scala_doc_sources = 'find . -type f -name "*.scala" | egrep \"\.\/core|\.\/infer\" | egrep -v \"\/javaapi\"  | egrep -v \"Suite\"'
     scala_doc_classpath = ':'.join([
         '`find native -name "*.jar" | grep "target/lib/" | tr "\\n" ":" `',
-        '`find macros -name "*-SNAPSHOT.jar" | tr "\\n" ":" `',
-        '`find core -name "*-SNAPSHOT.jar" | tr "\\n" ":" `',
-        '`find infer -name "*-SNAPSHOT.jar" | tr "\\n" ":" `'
+        '`find macros -name "*-INTERNAL.jar" | tr "\\n" ":" `',
+        '`find core -name "*-INTERNAL.jar" | tr "\\n" ":" `',
+        '`find infer -name "*-INTERNAL.jar" | tr "\\n" ":" `'
     ])
     # There are unresolvable errors on mxnet 1.2.x. We are ignoring those errors while aborting the ci on newer versions
     scala_ignore_errors = '; exit 0' if '1.2.' in _BUILD_VER else ''
@@ -135,9 +135,9 @@ def build_java_docs(app):
     java_doc_sources = 'find . -type f -name "*.scala" | egrep \"\.\/core|\.\/infer\" | egrep \"\/javaapi\" | egrep -v \"Suite\"'
     java_doc_classpath = ':'.join([
         '`find native -name "*.jar" | grep "target/lib/" | tr "\\n" ":" `',
-        '`find macros -name "*-SNAPSHOT.jar" | tr "\\n" ":" `',
-        '`find core -name "*-SNAPSHOT.jar" | tr "\\n" ":" `',
-        '`find infer -name "*-SNAPSHOT.jar" | tr "\\n" ":" `'
+        '`find macros -name "*-INTERNAL.jar" | tr "\\n" ":" `',
+        '`find core -name "*-INTERNAL.jar" | tr "\\n" ":" `',
+        '`find infer -name "*-INTERNAL.jar" | tr "\\n" ":" `'
     ])
     _run_cmd('cd {}; scaladoc `{}` -classpath {} -feature -deprecation'
              .format(java_path, java_doc_sources, java_doc_classpath))

diff --git a/scala-package/.gitignore b/scala-package/.gitignore
@@ -1,10 +1,10 @@
 .flattened-pom.xml
 core/src/main/scala/org/apache/mxnet/NDArrayAPIBase.scala
 core/src/main/scala/org/apache/mxnet/NDArrayBase.scala
+core/src/main/scala/org/apache/mxnet/NDArrayRandomAPIBase.scala
 core/src/main/scala/org/apache/mxnet/javaapi/NDArrayBase.scala
 core/src/main/scala/org/apache/mxnet/SymbolAPIBase.scala
 core/src/main/scala/org/apache/mxnet/SymbolBase.scala
-core/src/main/scala/org/apache/mxnet/NDArrayRandomAPIBase.scala
 core/src/main/scala/org/apache/mxnet/SymbolRandomAPIBase.scala
 examples/scripts/infer/images/
 examples/scripts/infer/models/

diff --git a/scala-package/README.md b/scala-package/README.md
@@ -128,7 +128,7 @@ You can also run the unit tests and integration tests on the Scala Package by :
 
 ```bash
 cd scala-package
-mvn integration-test
+mvn integration-test -DskipTests=false
 ```
 
 Or run a subset of unit tests, for e.g.,
@@ -141,6 +141,25 @@ mvn -DSCALA_TEST_ARGS=-Dsuites=org.apache.mxnet.NDArraySuite integration-test
 If everything goes well, you will find jars for `assembly`, `core` and `example` modules.
 Also it produces the native library in `native/target`, which you can use to cooperate with the `core` module.
 
+Deploy to repository
+--------------------
+
+By default, `maven deploy` will deploy artifacts to local file system, you can file then in: ``scala-package/deploy/target/repo`` folder.
+
+For nightly build in CI, a snapshot build will be uploaded to apache repository with follow command:
+
+```bash
+cd scala-package
+mvn deploy -Pnightly
+```
+
+Use following command to deploy release build (push artifacts to apache staging repository):
+
+```bash
+cd scala-package
+mvn deploy -Pstaging
+```
+
 Examples & Usage
 -------
 - To set up the Scala Project using IntelliJ IDE on macOS follow the instructions [here](https://mxnet.incubator.apache.org/tutorials/scala/mxnet_scala_on_intellij.html).

diff --git a/scala-package/assembly/pom.xml b/scala-package/assembly/pom.xml
@@ -6,140 +6,63 @@
   <parent>
     <groupId>org.apache.mxnet</groupId>
     <artifactId>mxnet-parent</artifactId>
-    <version>1.5.0-SNAPSHOT</version>
+    <version>INTERNAL</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
   <artifactId>mxnet-full_2.11</artifactId>
-  <name>MXNet Scala Package - Full ${platform}-only</name>
+  <name>Assembly Scala Package</name>
   <packaging>pom</packaging>
 
   <properties>
-    <MXNET_DIR>${parent.basedir}/..</MXNET_DIR>
+    <MXNET_DIR>${project.parent.basedir}/..</MXNET_DIR>
   </properties>
 
   <dependencies>
     <dependency>
       <groupId>org.apache.mxnet</groupId>
       <artifactId>mxnet-core</artifactId>
-      <version>1.5.0-SNAPSHOT</version>
+      <version>INTERNAL</version>
     </dependency>
     <dependency>
       <groupId>org.apache.mxnet</groupId>
       <artifactId>libmxnet-scala</artifactId>
-      <version>1.5.0-SNAPSHOT</version>
+      <version>INTERNAL</version>
       <type>${libtype}</type>
     </dependency>
     <dependency>
       <groupId>org.apache.mxnet</groupId>
       <artifactId>mxnet-infer</artifactId>
-      <version>1.5.0-SNAPSHOT</version>
+      <version>INTERNAL</version>
     </dependency>
   </dependencies>
 
   <profiles>
     <profile>
-      <id>release</id>
+      <id>staging</id>
       <build>
         <plugins>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-source-plugin</artifactId>
-            <executions>
-              <execution>
-                <phase>package</phase>
-                <goals>
-                  <goal>jar-no-fork</goal>
-                </goals>
-                <configuration>
-                  <includePom>true</includePom>
-                </configuration>
-              </execution>
-            </executions>
-          </plugin>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-javadoc-plugin</artifactId>
-            <executions>
-              <execution>
-                <phase>package</phase>
-                <goals>
-                  <goal>jar</goal>
-                </goals>
-                <configuration>
-                  <includeDependencySources>true</includeDependencySources>
-                  <dependencySourceExcludes>
-                    <dependencySourceExclude>commons-codec:*</dependencySourceExclude>
-                    <dependencySourceExclude>org.scala-lang:*</dependencySourceExclude>
-                    <dependencySourceExclude>log4j:*</dependencySourceExclude>
-                    <dependencySourceExclude>org.slf4j:*</dependencySourceExclude>
-                  </dependencySourceExcludes>
-                </configuration>
-              </execution>
-            </executions>
-          </plugin>
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-gpg-plugin</artifactId>
+            <version>1.6</version>
             <executions>
               <execution>
                 <id>sign-artifacts</id>
-                <phase>verify</phase>
+                <phase>deploy</phase>
                 <goals>
                   <goal>sign</goal>
                 </goals>
               </execution>
             </executions>
           </plugin>
-          <plugin>
-            <groupId>org.sonatype.plugins</groupId>
-            <artifactId>nexus-staging-maven-plugin</artifactId>
-            <extensions>true</extensions>
-            <configuration>
-              <serverId>ossrh</serverId>
-              <nexusUrl>https://oss.sonatype.org/</nexusUrl>
-              <autoReleaseAfterClose>true</autoReleaseAfterClose>
-            </configuration>
-          </plugin>
         </plugins>
       </build>
     </profile>
   </profiles>
 
   <build>
     <plugins>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>flatten-maven-plugin</artifactId>
-        <version>1.0.1</version>
-        <configuration>
-          <pomElements>
-            <dependencies>remove</dependencies>
-          </pomElements>
-        </configuration>
-        <executions>
-          <!-- enable flattening -->
-          <execution>
-            <id>flatten</id>
-            <phase>process-resources</phase>
-            <goals>
-              <goal>flatten</goal>
-            </goals>
-          </execution>
-          <!-- ensure proper cleanup -->
-          <execution>
-            <id>flatten.clean</id>
-            <phase>clean</phase>
-            <goals>
-              <goal>clean</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-install-plugin</artifactId>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-assembly-plugin</artifactId>
@@ -151,7 +74,6 @@
               <goal>single</goal>
             </goals>
             <configuration>
-              <finalName>${project.artifactId}-${platform}-${flavor}-${project.version}</finalName>
               <appendAssemblyId>false</appendAssemblyId>
               <descriptors>
                 <descriptor>src/main/assembly/assembly.xml</descriptor>
@@ -167,7 +89,7 @@
             <configuration>
               <appendAssemblyId>true</appendAssemblyId>
               <descriptors>
-                <descriptor>src/source.xml</descriptor>
+                <descriptor>src/main/assembly/source.xml</descriptor>
               </descriptors>
             </configuration>
           </execution>
@@ -180,13 +102,12 @@
             <configuration>
               <appendAssemblyId>true</appendAssemblyId>
               <descriptors>
-                <descriptor>src/javadoc.xml</descriptor>
+                <descriptor>src/main/assembly/javadoc.xml</descriptor>
               </descriptors>
             </configuration>
           </execution>
         </executions>
       </plugin>
     </plugins>
   </build>
-
 </project>