Support Databricks 3.0.1 with new build profiles

integration_tests/pom.xml

@@ -94,7 +94,9 @@
         <profile>
             <id>with-classifier</id>
             <activation>
-                <property><name>buildver</name></property>
+                <property>
+                    <name>buildver</name>
+                </property>
             </activation>
             <dependencies>
                 <dependency>
@@ -166,36 +168,41 @@
             </dependencies>
         </profile>
         <profile>
-            <id>release311db</id>
+            <!--
+                 Note that we are using the Spark version for all of the Databricks dependencies as well.
+                 The jenkins/databricks/build.sh script handles installing the jars as maven artifacts.
+                 This is to make it easier and not have to change version numbers for each individual dependency
+                 and deal with differences between Databricks versions
+            -->
+            <id>dbdeps</id>
             <activation>
                 <property>
-                    <name>buildver</name>
-                    <value>311db</value>
+                    <name>databricks</name>
                 </property>
             </activation>
             <dependencies>
                 <dependency>
                     <groupId>org.apache.spark</groupId>
                     <artifactId>spark-sql_${scala.binary.version}</artifactId>
-                    <version>${spark311db.version}</version>
+                    <version>${spark.version}</version>
                     <scope>provided</scope>
                 </dependency>
                 <dependency>
                     <groupId>org.apache.spark</groupId>
                     <artifactId>spark-core_${scala.binary.version}</artifactId>
-                    <version>${spark311db.version}</version>
+                    <version>${spark.version}</version>
                     <scope>provided</scope>
                 </dependency>
                 <dependency>
                     <groupId>org.apache.spark</groupId>
                     <artifactId>spark-annotation_${scala.binary.version}</artifactId>
-                    <version>${spark311db.version}</version>
+                    <version>${spark.version}</version>
                     <scope>provided</scope>
                 </dependency>
                 <dependency>
                     <groupId>org.apache.spark</groupId>
                     <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
-                    <version>${spark311db.version}</version>
+                    <version>${spark.version}</version>
                     <scope>provided</scope>
                 </dependency>
                 <dependency>
@@ -206,27 +213,27 @@
                 </dependency>
                 <dependency>
                     <groupId>org.apache.arrow</groupId>
-		    <artifactId>arrow-format</artifactId>
-                    <version>${spark311db.version}</version>
+                    <artifactId>arrow-format</artifactId>
+                    <version>${spark.version}</version>
                     <scope>compile</scope>
                 </dependency>
                 <dependency>
                     <groupId>org.apache.arrow</groupId>
-		    <artifactId>arrow-memory</artifactId>
-                    <version>${spark311db.version}</version>
+                    <artifactId>arrow-memory</artifactId>
+                    <version>${spark.version}</version>
                     <scope>compile</scope>
                 </dependency>
                 <dependency>
                     <groupId>org.apache.arrow</groupId>
-		    <artifactId>arrow-memory</artifactId>
-                    <version>${spark311db.version}</version>
+                    <artifactId>arrow-vector</artifactId>
+                    <version>${spark.version}</version>
                     <scope>compile</scope>
                 </dependency>
                 <dependency>
-                    <groupId>org.apache.arrow</groupId>
-		    <artifactId>arrow-vector</artifactId>
-                    <version>${spark311db.version}</version>
-                    <scope>compile</scope>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-annotations</artifactId>
+                    <version>${spark.version}</version>
+                    <scope>provided</scope>
                 </dependency>
             </dependencies>
         </profile>

jenkins/databricks/build.sh

@@ -94,31 +94,47 @@ PARQUETCOMMONJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-
 PARQUETCOLUMNJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-column--org.apache.parquet__parquet-column__1.10.1-databricks6.jar
 PARQUETFORMATJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-format--org.apache.parquet__parquet-format__2.4.0.jar
 
-ARROWFORMATJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-format--org.apache.arrow__arrow-format__2.0.0.jar
-ARROWMEMORYJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-memory-core--org.apache.arrow__arrow-memory-core__2.0.0.jar
-ARROWMEMORYNETTYJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-memory-netty--org.apache.arrow__arrow-memory-netty__2.0.0.jar
-ARROWVECTORJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-vector--org.apache.arrow__arrow-vector__2.0.0.jar
-
 NETWORKCOMMON=----workspace_${SPARK_MAJOR_VERSION_STRING}--common--network-common--network-common-hive-2.3__hadoop-2.7_2.12_deploy.jar
 COMMONUNSAFE=----workspace_${SPARK_MAJOR_VERSION_STRING}--common--unsafe--unsafe-hive-2.3__hadoop-2.7_2.12_deploy.jar
 LAUNCHER=----workspace_${SPARK_MAJOR_VERSION_STRING}--launcher--launcher-hive-2.3__hadoop-2.7_2.12_deploy.jar
 
 KRYO=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--com.esotericsoftware--kryo-shaded--com.esotericsoftware__kryo-shaded__4.0.2.jar
 
 APACHECOMMONS=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--commons-io--commons-io--commons-io__commons-io__2.4.jar
-APACHECOMMONSLANG3=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-lang3--org.apache.commons__commons-lang3__3.10.jar
 
-JSON4S=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.json4s--json4s-ast_2.12--org.json4s__json4s-ast_2.12__3.7.0-M5.jar
+if [[ $BASE_SPARK_VERSION == "3.0.1" ]]
+then
+    JSON4S=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.json4s--json4s-ast_2.12--org.json4s__json4s-ast_2.12__3.6.6.jar
+    APACHECOMMONSLANG3=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-lang3--org.apache.commons__commons-lang3__3.9.jar
+    HIVESTORAGE=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-storage-api--org.apache.hive__hive-storage-api__2.7.1.jar
+    ARROWFORMATJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-format--org.apache.arrow__arrow-format__0.15.1.jar
+    ARROWMEMORYJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-memory--org.apache.arrow__arrow-memory__0.15.1.jar
+    ARROWVECTORJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-vector--org.apache.arrow__arrow-vector__0.15.1.jar
+    HIVEEXECJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--patched-hive-with-glue--hive-exec-core_shaded.jar
+else
+    JSON4S=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.json4s--json4s-ast_2.12--org.json4s__json4s-ast_2.12__3.7.0-M5.jar
+    APACHECOMMONSLANG3=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-lang3--org.apache.commons__commons-lang3__3.10.jar
+    HIVESTORAGE=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-storage-api--org.apache.hive__hive-storage-api__2.7.2.jar
+    HIVEEXECJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-exec-core--org.apache.hive__hive-exec-core__2.3.7.jar
+    ARROWFORMATJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-format--org.apache.arrow__arrow-format__2.0.0.jar
+    ARROWMEMORYJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-memory-core--org.apache.arrow__arrow-memory-core__2.0.0.jar
+    ARROWMEMORYNETTYJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-memory-netty--org.apache.arrow__arrow-memory-netty__2.0.0.jar
+    ARROWVECTORJAR=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-vector--org.apache.arrow__arrow-vector__2.0.0.jar
+fi
 
 JAVAASSIST=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.javassist--javassist--org.javassist__javassist__3.25.0-GA.jar
 
 PROTOBUFJAVA=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--com.google.protobuf--protobuf-java--com.google.protobuf__protobuf-java__2.6.1.jar
 
 JACKSONCORE=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.core--jackson-databind--com.fasterxml.jackson.core__jackson-databind__2.10.0.jar
+JACKSONANNOTATION=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.core--jackson-annotations--com.fasterxml.jackson.core__jackson-annotations__2.10.0.jar
 
 HADOOPCOMMON=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-common--org.apache.hadoop__hadoop-common__2.7.4.jar
 HADOOPMAPRED=----workspace_${SPARK_MAJOR_VERSION_STRING}--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-mapreduce-client-core--org.apache.hadoop__hadoop-mapreduce-client-core__2.7.4.jar
 
+# Please note we are installing all of these dependencies using the Spark version (SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS) to make it easier
+# to specify the dependencies in the pom files
+
 mvn -B install:install-file \
    -Dmaven.repo.local=$M2DIR \
    -Dfile=$JARDIR/$CATALYSTJAR \
@@ -161,6 +177,14 @@ then
 
 else
 
+    mvn -B install:install-file \
+       -Dmaven.repo.local=$M2DIR \
+       -Dfile=$JARDIR/$HIVEEXECJAR \
+       -DgroupId=org.apache.hive \
+       -DartifactId=hive-exec \
+       -Dversion=$SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS \
+       -Dpackaging=jar
+
     mvn -B install:install-file \
        -Dmaven.repo.local=$M2DIR \
        -Dfile=$JARDIR/$COREJAR \
@@ -177,6 +201,14 @@ else
        -Dversion=$SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS \
        -Dpackaging=jar
 
+    mvn -B install:install-file \
+       -Dmaven.repo.local=$M2DIR \
+       -Dfile=$JARDIR/$JACKSONANNOTATION\
+       -DgroupId=com.fasterxml.jackson.core \
+       -DartifactId=jackson-annotations \
+       -Dversion=$SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS \
+       -Dpackaging=jar
+
     mvn -B install:install-file \
        -Dmaven.repo.local=$M2DIR \
        -Dfile=$JARDIR/$PROTOBUFJAVA \
@@ -265,14 +297,6 @@ else
        -Dversion=$SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS \
        -Dpackaging=jar
 
-    mvn -B install:install-file \
-       -Dmaven.repo.local=$M2DIR \
-       -Dfile=$JARDIR/$HIVEEXECJAR \
-       -DgroupId=org.apache.hive \
-       -DartifactId=hive-exec \
-       -Dversion=$SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS \
-       -Dpackaging=jar
-
     mvn -B install:install-file \
        -Dmaven.repo.local=$M2DIR \
        -Dfile=$JARDIR/$HIVESERDEJAR \
@@ -329,14 +353,6 @@ else
        -Dversion=$SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS \
        -Dpackaging=jar
 
-    mvn -B install:install-file \
-       -Dmaven.repo.local=$M2DIR \
-       -Dfile=$JARDIR/$ARROWMEMORYNETTYJAR \
-       -DgroupId=org.apache.arrow \
-       -DartifactId=arrow-memory-netty \
-       -Dversion=$SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS \
-       -Dpackaging=jar
-
     mvn -B install:install-file \
        -Dmaven.repo.local=$M2DIR \
        -Dfile=$JARDIR/$ARROWVECTORJAR \

pom.xml

@@ -193,6 +193,34 @@
             </modules>
         </profile>
         <profile>
+            <!-- Note databricks requires 2 properties -Ddatabricks and -Dbuildver=[301db,311db] -->
+            <id>release301db</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>301db</value>
+                </property>
+            </activation>
+            <properties>
+                <!-- Downgrade scala plugin version due to: https://github.com/sbt/sbt/issues/4305 -->
+                <scala.plugin.version>3.4.4</scala.plugin.version>
+                <shim.module.name>spark301db</shim.module.name>
+                <spark.version.classifier>spark301-databricks</spark.version.classifier>
+                <!--
+                     Note that we are using the Spark version for all of the Databricks dependencies as well.
+                     The jenkins/databricks/build.sh script handles installing the jars as maven artifacts.
+                     This is to make it easier and not have to change version numbers for each individual dependency
+                     and deal with differences between Databricks versions
+                -->
+                <spark.version>${spark301db.version}</spark.version>
+                <spark.test.version>${spark301db.version}</spark.test.version>
+                <rat.consoleOutput>true</rat.consoleOutput>
+                <hive.storage.api.version>${spark301db.version}</hive.storage.api.version>
+                <protobuf.java.version>${spark301db.version}</protobuf.java.version>
+            </properties>
+        </profile>
+        <profile>
+            <!-- Note databricks requires 2 properties -Ddatabricks and -Dbuildver=[301db,311db] -->
             <id>release311db</id>
             <activation>
                 <property>
@@ -201,8 +229,16 @@
                 </property>
             </activation>
             <properties>
+                <!-- Downgrade scala plugin version due to: https://github.com/sbt/sbt/issues/4305 -->
+                <scala.plugin.version>3.4.4</scala.plugin.version>
                 <shim.module.name>spark311db</shim.module.name>
                 <spark.version.classifier>spark311-databricks</spark.version.classifier>
+                <!--
+                     Note that we are using the Spark version for all of the Databricks dependencies as well.
+                     The jenkins/databricks/build.sh script handles installing the jars as maven artifacts.
+                     This is to make it easier and not have to change version numbers for each individual dependency
+                     and deal with differences between Databricks versions
+                -->
                 <spark.version>${spark311db.version}</spark.version>
                 <spark.test.version>${spark311db.version}</spark.test.version>
                 <rat.consoleOutput>true</rat.consoleOutput>