Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ADD 321cdh shim to rapids and remove 311cdh shim #5346

Merged
merged 7 commits into from
May 2, 2022
Merged

ADD 321cdh shim to rapids and remove 311cdh shim #5346

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
19f739d
ADD 321cdh shim
Apr 11, 2022
71c3088
Address review comments 1.
Apr 29, 2022
11ae655
Move ORC shim to 321cdh directory
Apr 29, 2022
8f7075f
Remove ArrowShims and fix style errors
Apr 29, 2022
a038bf3
address review comments
sririshindra Apr 29, 2022
4d11395
address review comments
sririshindra Apr 29, 2022
0e96303
address latest review comments
sririshindra May 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 39 additions & 8 deletions api_validation/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,31 +68,62 @@
</dependencies>
</profile>
<profile>
<id>release311cdh</id>
<id>release321cdh</id>
<activation>
<property>
<name>buildver</name>
<value>311cdh</value>
<value>321cdh</value>
</property>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark311cdh.version}</version>
<version>${spark321cdh.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
</exclusion>
</exclusions>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>4.3.0.7.2.7.0-184</version>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark321cdh.version}</version>
<exclusions>
<!-- spark-core tries to pull a curator-recipes version we don't want -->
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
</exclusion>
</exclusions>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
tgravescs marked this conversation as resolved.
Show resolved Hide resolved
<artifactId>arrow-vector</artifactId>
<version>${arrow.cdh.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
Expand Down
6 changes: 3 additions & 3 deletions build/buildall
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ case $DIST_PROFILE in
snapshots?(WithDatabricks))
SPARK_SHIM_VERSIONS=(
311
311cdh
321cdh
312
313
314
Expand All @@ -153,7 +153,7 @@ case $DIST_PROFILE in
noSnapshots?(WithDatabricks))
SPARK_SHIM_VERSIONS=(
311
311cdh
321cdh
312
313
320
Expand All @@ -163,7 +163,7 @@ case $DIST_PROFILE in

minimumFeatureVersionMix)
SPARK_SHIM_VERSIONS=(
311cdh
321cdh
312
320
)
Expand Down
57 changes: 44 additions & 13 deletions dist/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@
<target.classifier/>
<noSnapshot.buildvers>
311,
311cdh,
312,
313,
320,
321
321,
321cdh
</noSnapshot.buildvers>
<snapshot.buildvers>
314,
Expand Down Expand Up @@ -98,7 +98,7 @@
<included_buildvers>
312,
320,
311cdh
321cdh
</included_buildvers>
</properties>
</profile>
Expand All @@ -122,11 +122,11 @@
</properties>
</profile>
<profile>
<id>release311cdh</id>
<id>release321cdh</id>
<activation>
<property>
<name>buildver</name>
<value>311cdh</value>
<value>321cdh</value>
</property>
</activation>
<properties>
Expand All @@ -136,28 +136,59 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark311cdh.version}</version>
<version>${spark321cdh.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
</exclusion>
</exclusions>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>4.3.0.7.2.7.0-184</version>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark321cdh.version}</version>
<exclusions>
<!-- spark-core tries to pull a curator-recipes version we don't want -->
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
</exclusion>
</exclusions>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
<version>${arrow.cdh.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<pluginRepositories>
sririshindra marked this conversation as resolved.
Show resolved Hide resolved
<pluginRepositories>
<pluginRepository>
<id>cloudera-repo</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</pluginRepository>
tgravescs marked this conversation as resolved.
Show resolved Hide resolved
</pluginRepositories>
</pluginRepositories>
</profile>
<profile>
<id>individual</id>
Expand Down
51 changes: 32 additions & 19 deletions integration_tests/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,48 +69,61 @@

<profiles>
<profile>
<id>release311cdh</id>
<id>release321cdh</id>
<activation>
<property>
<name>buildver</name>
<value>311cdh</value>
<value>321cdh</value>
</property>
</activation>
<dependencies>
<dependency>
<!-- it seems CDH jars pull an old version of Guava but need new version
to run with, so here we override Guava version for testing -->
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.cdh.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark311cdh.version}</version>
<version>${spark321cdh.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
</exclusion>
</exclusions>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>4.3.0.7.2.7.0-184</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark311cdh.version}</version>
<version>${spark321cdh.version}</version>
<exclusions>
<!-- spark-core tries to pull a curator-recipes version we don't want -->
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
</exclusion>
</exclusions>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
<version>${arrow.cdh.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
Expand Down
2 changes: 1 addition & 1 deletion jenkins/spark-premerge-build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ mvn_verify() {

# build all the versions but only run unit tests on one 3.1.X version (base version covers this), and one 3.2.X version.
# All others shims test should be covered in nightly pipelines
env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Dbuildver=311cdh clean install -Drat.skip=true -DskipTests -Dmaven.javadoc.skip=true -Dskip -Dmaven.scalastyle.skip=true -Dcuda.version=$CUDA_CLASSIFIER -pl aggregator -am
env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Dbuildver=321cdh clean install -Drat.skip=true -DskipTests -Dmaven.javadoc.skip=true -Dskip -Dmaven.scalastyle.skip=true -Dcuda.version=$CUDA_CLASSIFIER -pl aggregator -am
env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Dbuildver=312 clean install -Drat.skip=true -DskipTests -Dmaven.javadoc.skip=true -Dskip -Dmaven.scalastyle.skip=true -Dcuda.version=$CUDA_CLASSIFIER -pl aggregator -am
env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Dbuildver=313 clean install -Drat.skip=true -DskipTests -Dmaven.javadoc.skip=true -Dskip -Dmaven.scalastyle.skip=true -Dcuda.version=$CUDA_CLASSIFIER -pl aggregator -am
[[ $BUILD_SNAPSHOTS == "true" ]] && env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Dbuildver=314 clean install -Drat.skip=true -DskipTests -Dmaven.javadoc.skip=true -Dskip -Dmaven.scalastyle.skip=true -Dcuda.version=$CUDA_CLASSIFIER -pl aggregator -am
Expand Down
2 changes: 1 addition & 1 deletion jenkins/version-def.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ echo "CUDF_VER: $CUDF_VER, CUDA_CLASSIFIER: $CUDA_CLASSIFIER, PROJECT_VER: $PROJ
SPARK_VER: $SPARK_VER, SCALA_BINARY_VER: $SCALA_BINARY_VER"


SPARK_SHIM_VERSIONS_STR=${SPARK_SHIM_VERSIONS_STR:-"311 311cdh 312 313 314 320 321 322 330"}
SPARK_SHIM_VERSIONS_STR=${SPARK_SHIM_VERSIONS_STR:-"311 321cdh 312 313 314 320 321 322 330"}

IFS=" " <<< $SPARK_SHIM_VERSIONS_STR read -r -a SPARK_SHIM_VERSIONS

Expand Down
Loading