Change deploy scripts for new build system (#3683)

* Change deploy script to release all classifier integration test jars Signed-off-by: Thomas Graves <tgraves@nvidia.com> * Update databricks deploy to push integration test jar * update buildver for 301 * Some dist profiles missing use base version * fixes Signed-off-by: Thomas Graves <tgraves@nvidia.com> * Change path to tools jars Signed-off-by: Thomas Graves <tgraves@nvidia.com>
NVIDIA · Sep 28, 2021 · d19fb1f · d19fb1f
1 parent e149f4c
commit d19fb1f
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 9 deletions.
diff --git a/jenkins/databricks/deploy.sh b/jenkins/databricks/deploy.sh
@@ -34,3 +34,7 @@ DBJARFPATH=./aggregator/target/rapids-4-spark-aggregator_$SCALA_VERSION-$SPARK_P
 echo "Databricks jar is: $DBJARFPATH"
 mvn -B deploy:deploy-file $MVN_URM_MIRROR -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
     -Dfile=$DBJARFPATH -DpomFile=aggregator/dependency-reduced-pom.xml -Dclassifier=$DB_SHIM_NAME
+# install the integration test jar
+DBINTTESTJARFPATH=./integration_tests/target/rapids-4-spark-integration-tests_$SCALA_VERSION-$SPARK_PLUGIN_JAR_VERSION-${DB_SHIM_NAME}.jar
+mvn -B deploy:deploy-file $MVN_URM_MIRROR -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
+    -Dfile=$DBINTTESTJARFPATH -DpomFile=integration_tests/pom.xml -Dclassifier=$DB_SHIM_NAME
diff --git a/jenkins/deploy.sh b/jenkins/deploy.sh
@@ -18,18 +18,22 @@
 # Argument(s):
 #   SIGN_FILE:  true/false, whether to sign the jar/pom file to de deployed
 #   DATABRICKS: true/fasle, whether deploying for databricks
+#   VERSIONS_BUILT: The spark versions built before calling this script
 #
 # Used environment(s):
 #   SQL_PL:         The path of module 'sql-plugin', relative to project root path.
 #   DIST_PL:        The path of module 'dist', relative to project root path.
+#   AGGREGATOR_PL:  The path of the module 'aggregator', relative to project root path.
+#   TESTS_PL:       The path of the module 'integration_tests', relative to the project root path.
 #   SERVER_ID:      The repository id for this deployment.
 #   SERVER_URL:     The url where to deploy artifacts.
 #   GPG_PASSPHRASE: The passphrase used to sign files, only required when <SIGN_FILE> is true.
 ###
 
-set -e
+set -ex
 SIGN_FILE=$1
 DATABRICKS=$2
+VERSIONS_BUILT=$3
 
 ###### Build the path of jar(s) to be deployed ######
 
@@ -78,22 +82,25 @@ $DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
 # Distribution jar is a shaded artifact so use the reduced dependency pom.
 $DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
             $SRC_DOC_JARS \
-            -Dfile=$FPATH.jar -DpomFile=${DIST_PL}/dependency-reduced-pom.xml
+            -Dfile=$FPATH.jar -DgroupId=com.nvidia -DartifactId=$ART_ID -Dversion=$ART_VER
 
 ###### Deploy integration tests jar(s) ######
 TESTS_ART_ID=`mvn help:evaluate -q -pl $TESTS_PL -Dexpression=project.artifactId -DforceStdout`
 TESTS_ART_VER=`mvn help:evaluate -q -pl $TESTS_PL -Dexpression=project.version -DforceStdout`
-TESTS_FPATH="$TESTS_PL/target/$TESTS_ART_ID-$TESTS_ART_VER"
-TESTS_DOC_JARS="-Dsources=${TESTS_FPATH}-sources.jar -Djavadoc=${TESTS_FPATH}-javadoc.jar"
-$DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
+TESTS_DOC_JARS="-Dsources=deployjars/$TESTS_ART_ID-$TESTS_ART_VER-sources.jar -Djavadoc=deployjars/$TESTS_ART_ID-$TESTS_ART_VER-javadoc.jar"
+VERSIONS_LIST=${VERSIONS_BUILT//','/' '}
+for VER in ${VERSIONS_LIST}; do
+    TESTS_FPATH="deployjars/$TESTS_ART_ID-$TESTS_ART_VER-spark$VER"
+    $DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
             $TESTS_DOC_JARS \
-            -Dfile=$TESTS_FPATH.jar -DpomFile=${TESTS_PL}/pom.xml
+            -Dfile=$TESTS_FPATH.jar -DpomFile=${TESTS_PL}/pom.xml -Dclassifier=spark$VER
+done
 
 ###### Deploy profiling tool jar(s) ######
 TOOL_PL=${TOOL_PL:-"tools"}
-TOOL_ART_ID=`mvn help:evaluate -q -pl $TOOL_PL -Dexpression=project.artifactId -DforceStdout`
-TOOL_ART_VER=`mvn help:evaluate -q -pl $TOOL_PL -Dexpression=project.version -DforceStdout`
-TOOL_FPATH="$TOOL_PL/target/$TOOL_ART_ID-$TOOL_ART_VER"
+TOOL_ART_ID=`mvn help:evaluate -q -pl $TOOL_PL -Dexpression=project.artifactId -DforceStdout -Prelease311`
+TOOL_ART_VER=`mvn help:evaluate -q -pl $TOOL_PL -Dexpression=project.version -DforceStdout -Prelease311`
+TOOL_FPATH="deployjars/$TOOL_ART_ID-$TOOL_ART_VER"
 TOOL_DOC_JARS="-Dsources=${TOOL_FPATH}-sources.jar -Djavadoc=${TOOL_FPATH}-javadoc.jar"
 $DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
             $TOOL_DOC_JARS \

diff --git a/pom.xml b/pom.xml
@@ -85,6 +85,10 @@
             <id>release301</id>
             <activation>
                 <activeByDefault>true</activeByDefault>
+                <property>
+                    <name>buildver</name>
+                    <value>301</value>
+                </property>
             </activation>
             <properties>
                 <spark.version>${spark301.version}</spark.version>