From a7bab25fd0b13ea9f893bafc0a1ebaf73b9f0b81 Mon Sep 17 00:00:00 2001 From: Chong Gao Date: Mon, 7 Mar 2022 10:58:55 +0800 Subject: [PATCH 1/2] Revert "Integration-test tests jar for hive UDF tests (#4890)" This reverts commit 1d0004ca87e04e8435d3390bc59f47f602c177b3. --- integration_tests/pom.xml | 17 ----------------- jenkins/databricks/deploy.sh | 4 ---- jenkins/spark-tests.sh | 18 +++++------------- 3 files changed, 5 insertions(+), 34 deletions(-) diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml index 9c6607adada..eef75b9c1bc 100644 --- a/integration_tests/pom.xml +++ b/integration_tests/pom.xml @@ -248,23 +248,6 @@ run - - generate-test-build-info - process-test-classes - - - - - - - - - - - - run - - diff --git a/jenkins/databricks/deploy.sh b/jenkins/databricks/deploy.sh index a3530d3a9f4..6583c03871a 100755 --- a/jenkins/databricks/deploy.sh +++ b/jenkins/databricks/deploy.sh @@ -38,7 +38,3 @@ mvn -B deploy:deploy-file $MVN_URM_MIRROR -Durl=$SERVER_URL -DrepositoryId=$SERV DBINTTESTJARFPATH=./integration_tests/target/rapids-4-spark-integration-tests_$SCALA_VERSION-$SPARK_PLUGIN_JAR_VERSION-${DB_SHIM_NAME}.jar mvn -B deploy:deploy-file $MVN_URM_MIRROR -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \ -Dfile=$DBINTTESTJARFPATH -DpomFile=integration_tests/pom.xml -Dclassifier=$DB_SHIM_NAME -# Deploy integration-tests tests jar for hive UDF tests -HIVEUDFTESTSJAR=./integration_tests/target/rapids-4-spark-integration-tests_$SCALA_VERSION-$SPARK_PLUGIN_JAR_VERSION-${DB_SHIM_NAME}'tests'.jar -mvn -B deploy:deploy-file $MVN_URM_MIRROR -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \ - -Dfile=$HIVEUDFTESTSJAR -DpomFile=integration_tests/pom.xml -Dclassifier=$DB_SHIM_NAME'tests' diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh index 935dd420ffa..1932365ee23 100755 --- a/jenkins/spark-tests.sh +++ b/jenkins/spark-tests.sh @@ -54,12 +54,6 @@ RAPIDS_INT_TESTS_HOME="$ARTF_ROOT/integration_tests/" # The version of pytest.tar.gz that is uploaded is the one built against spark301 but its being pushed without classifier for now RAPIDS_INT_TESTS_TGZ="$ARTF_ROOT/rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-$PROJECT_TEST_VER-pytest.tar.gz" -# For the HIVE UDF integration tests -$MVN_GET_CMD -DremoteRepositories=$PROJECT_TEST_REPO \ - -Dtransitive=false \ - -DgroupId=com.nvidia -DartifactId=rapids-4-spark-integration-tests_$SCALA_BINARY_VER -Dversion=$PROJECT_TEST_VER -Dclassifier=$SHUFFLE_SPARK_SHIM'tests' -RAPIDS_HIVE_UDF_TESTS_JAR=="$ARTF_ROOT/rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-$PROJECT_TEST_VER-$SHUFFLE_SPARK_SHIM'tests'.jar" - tmp_info=${TMP_INFO_FILE:-'/tmp/artifacts-build.info'} rm -rf "$tmp_info" TEE_CMD="tee -a $tmp_info" @@ -80,22 +74,20 @@ getRevision() { set +x echo -e "\n==================== ARTIFACTS BUILD INFO ====================\n" >> "$tmp_info" echo "-------------------- cudf JNI BUILD INFO --------------------" >> "$tmp_info" -c_ver=$(getRevision $CUDF_JAR cudf-java-version-info.properties) +c_ver=$(getRevision $JARS_PATH/$CUDF_JAR cudf-java-version-info.properties) echo "-------------------- rapids-4-spark BUILD INFO --------------------" >> "$tmp_info" -p_ver=$(getRevision $RAPIDS_PLUGIN_JAR rapids4spark-version-info.properties) +p_ver=$(getRevision $JARS_PATH/$RAPIDS_PLUGIN_JAR rapids4spark-version-info.properties) echo "-------------------- rapids-4-spark-integration-tests BUILD INFO --------------------" >> "$tmp_info" -it_ver=$(getRevision $RAPIDS_TEST_JAR rapids4spark-version-info.properties) +it_ver=$(getRevision $JARS_PATH/$RAPIDS_TEST_JAR rapids4spark-version-info.properties) echo "-------------------- rapids-4-spark-integration-tests pytest BUILD INFO --------------------" >> "$tmp_info" -pt_ver=$(getRevision $RAPIDS_INT_TESTS_TGZ integration_tests/rapids4spark-version-info.properties) -echo "-------------------- rapids-4-spark-integration-tests tests BUILD INFO --------------------" >> "$tmp_info" -hu_ver=$(getRevision $RAPIDS_HIVE_UDF_TESTS_JAR rapids4spark-version-info.properties) +pt_ver=$(getRevision $JARS_PATH/$RAPIDS_INT_TESTS_TGZ integration_tests/rapids4spark-version-info.properties) echo -e "\n==================== ARTIFACTS BUILD INFO ====================\n" >> "$tmp_info" set -x cat "$tmp_info" || true SKIP_REVISION_CHECK=${SKIP_REVISION_CHECK:-'false'} if [[ "$SKIP_REVISION_CHECK" != "true" && (-z "$c_ver" || -z "$p_ver"|| \ - "$p_ver" != "$it_ver" || "$p_ver" != "$pt_ver" || "$p_ver" != "$hu_ver") ]]; then + "$p_ver" != "$it_ver" || "$p_ver" != "$pt_ver") ]]; then echo "Artifacts revisions are inconsistent!" exit 1 fi From a28b6291015e5500154b5657d4beb68c391fff40 Mon Sep 17 00:00:00 2001 From: Chong Gao Date: Mon, 7 Mar 2022 10:58:39 +0800 Subject: [PATCH 2/2] Move Hive UDFs from test jar to main jar Signed-off-by: Chong Gao --- integration_tests/run_pyspark_from_build.sh | 8 ++++++-- .../spark/rapids/tests/udf/hive/EmptyHiveGenericUDF.java | 0 .../spark/rapids/tests/udf/hive/EmptyHiveSimpleUDF.java | 0 jenkins/spark-tests.sh | 8 ++++---- 4 files changed, 10 insertions(+), 6 deletions(-) rename integration_tests/src/{test => main}/java/com/nvidia/spark/rapids/tests/udf/hive/EmptyHiveGenericUDF.java (100%) rename integration_tests/src/{test => main}/java/com/nvidia/spark/rapids/tests/udf/hive/EmptyHiveSimpleUDF.java (100%) diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh index 217f2625bf4..867c1129341 100755 --- a/integration_tests/run_pyspark_from_build.sh +++ b/integration_tests/run_pyspark_from_build.sh @@ -41,12 +41,16 @@ else if [ -d "$LOCAL_JAR_PATH" ]; then CUDF_JARS=$(echo "$LOCAL_JAR_PATH"/cudf-*.jar) PLUGIN_JARS=$(echo "$LOCAL_JAR_PATH"/rapids-4-spark_*.jar) - TEST_JARS=$(echo "$LOCAL_JAR_PATH"/rapids-4-spark-integration-tests*-$SPARK_SHIM_VER*.jar) + # the integration-test-spark3xx.jar, should not include the integration-test-spark3xxtest.jar + TEST_JARS=$(echo "$LOCAL_JAR_PATH"/rapids-4-spark-integration-tests*-$SPARK_SHIM_VER.jar) else CUDF_JARS=$(echo "$SCRIPTPATH"/target/dependency/cudf-*.jar) PLUGIN_JARS=$(echo "$SCRIPTPATH"/../dist/target/rapids-4-spark_*.jar) - TEST_JARS=$(echo "$SCRIPTPATH"/target/rapids-4-spark-integration-tests*-$SPARK_SHIM_VER*.jar) + # the integration-test-spark3xx.jar, should not include the integration-test-spark3xxtest.jar + TEST_JARS=$(echo "$SCRIPTPATH"/target/rapids-4-spark-integration-tests*-$SPARK_SHIM_VER.jar) fi + + # Only 3 jars: cudf.jar dist.jar integration-test.jar ALL_JARS="$CUDF_JARS $PLUGIN_JARS $TEST_JARS" echo "AND PLUGIN JARS: $ALL_JARS" if [[ "${TEST}" != "" ]]; diff --git a/integration_tests/src/test/java/com/nvidia/spark/rapids/tests/udf/hive/EmptyHiveGenericUDF.java b/integration_tests/src/main/java/com/nvidia/spark/rapids/tests/udf/hive/EmptyHiveGenericUDF.java similarity index 100% rename from integration_tests/src/test/java/com/nvidia/spark/rapids/tests/udf/hive/EmptyHiveGenericUDF.java rename to integration_tests/src/main/java/com/nvidia/spark/rapids/tests/udf/hive/EmptyHiveGenericUDF.java diff --git a/integration_tests/src/test/java/com/nvidia/spark/rapids/tests/udf/hive/EmptyHiveSimpleUDF.java b/integration_tests/src/main/java/com/nvidia/spark/rapids/tests/udf/hive/EmptyHiveSimpleUDF.java similarity index 100% rename from integration_tests/src/test/java/com/nvidia/spark/rapids/tests/udf/hive/EmptyHiveSimpleUDF.java rename to integration_tests/src/main/java/com/nvidia/spark/rapids/tests/udf/hive/EmptyHiveSimpleUDF.java diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh index 1932365ee23..b2e068f6972 100755 --- a/jenkins/spark-tests.sh +++ b/jenkins/spark-tests.sh @@ -74,13 +74,13 @@ getRevision() { set +x echo -e "\n==================== ARTIFACTS BUILD INFO ====================\n" >> "$tmp_info" echo "-------------------- cudf JNI BUILD INFO --------------------" >> "$tmp_info" -c_ver=$(getRevision $JARS_PATH/$CUDF_JAR cudf-java-version-info.properties) +c_ver=$(getRevision $CUDF_JAR cudf-java-version-info.properties) echo "-------------------- rapids-4-spark BUILD INFO --------------------" >> "$tmp_info" -p_ver=$(getRevision $JARS_PATH/$RAPIDS_PLUGIN_JAR rapids4spark-version-info.properties) +p_ver=$(getRevision $RAPIDS_PLUGIN_JAR rapids4spark-version-info.properties) echo "-------------------- rapids-4-spark-integration-tests BUILD INFO --------------------" >> "$tmp_info" -it_ver=$(getRevision $JARS_PATH/$RAPIDS_TEST_JAR rapids4spark-version-info.properties) +it_ver=$(getRevision $RAPIDS_TEST_JAR rapids4spark-version-info.properties) echo "-------------------- rapids-4-spark-integration-tests pytest BUILD INFO --------------------" >> "$tmp_info" -pt_ver=$(getRevision $JARS_PATH/$RAPIDS_INT_TESTS_TGZ integration_tests/rapids4spark-version-info.properties) +pt_ver=$(getRevision $RAPIDS_INT_TESTS_TGZ integration_tests/rapids4spark-version-info.properties) echo -e "\n==================== ARTIFACTS BUILD INFO ====================\n" >> "$tmp_info" set -x cat "$tmp_info" || true