Prepare for 3.2.1 shim, various shim build fixes and improvements [databricks]

aggregator/pom.xml

@@ -63,6 +63,23 @@
     </dependencies>
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>${maven.jar.plugin.version}</version>
+                <configuration>
+                    <archive>
+                        <!-- transient jar, writing compressed can take several x time -->
+                        <compress>false</compress>
+                    </archive>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>default-test-jar</id>
+                        <phase>none</phase>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
@@ -120,41 +137,75 @@
                             </excludes>
                         </filter>
                     </filters>
+                    <createDependencyReducedPom>false</createDependencyReducedPom>
                 </configuration>
                 <executions>
-                    <!-- Unfortunately have to have 2 executions here to get dependency reduced pom.
-                         The shade plugin won't generate it when using the classifier and shadedArtifactAttached=true.
-                    -->
                     <execution>
-                        <id>main</id>
+                        <id>main-${spark.version.classifier}</id>
                         <phase>package</phase>
                         <goals>
                             <goal>shade</goal>
                         </goals>
-                        <configuration>
-                            <createDependencyReducedPom>true</createDependencyReducedPom>
-                            <shadedArtifactAttached>false</shadedArtifactAttached>
-                        </configuration>
                     </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.jacoco</groupId>
+                <artifactId>jacoco-maven-plugin</artifactId>
+                <executions>
                     <execution>
-                        <id>classifierversion</id>
+                        <id>prepare-agent</id>
+                        <goals>
+                            <goal>prepare-agent</goal>
+                        </goals>
+                        <phase>none</phase>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>default-test</id>
+                        <phase>none</phase>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>add-profile-src-30+</id>
+                        <phase>none</phase>
+                    </execution>
+                    <execution>
+                        <id>add-profile-src-31+</id>
+                        <phase>none</phase>
+                    </execution>
+                    <execution>
+                        <!-- WORKAROUND: default-jar does not contain classes by design.
+                             there seems to be a race between shade and empty default-jar
+                             So attaching here manually because maven complains when we try
+                             to disable default-jar to resolve the conflict.-->
+                        <id>attach-shaded-with-spark-version</id>
                         <phase>package</phase>
                         <goals>
-                            <goal>shade</goal>
+                            <goal>attach-artifact</goal>
                         </goals>
                         <configuration>
-                            <!-- note that the classifier version won't generate dependency reduce pom due to shadedArtifactAttached=true -->
-                            <createDependencyReducedPom>true</createDependencyReducedPom>
-                            <shadedArtifactAttached>true</shadedArtifactAttached>
-                            <shadedClassifierName>${spark.version.classifier}</shadedClassifierName>
+                            <artifacts>
+                                <artifact>
+                                    <file>${project.build.directory}/${project.artifactId}-${project.version}.jar</file>
+                                    <type>jar</type>
+                                    <classifier>${spark.version.classifier}</classifier>
+                                </artifact>
+                            </artifacts>
                         </configuration>
                     </execution>
                 </executions>
             </plugin>
-            <plugin>
-                <groupId>net.alchim31.maven</groupId>
-                <artifactId>scala-maven-plugin</artifactId>
-            </plugin>
             <plugin>
                 <groupId>org.apache.rat</groupId>
                 <artifactId>apache-rat-plugin</artifactId>

build/buildall

@@ -17,38 +17,56 @@
 
 set -e
 
+shopt -s extglob
+
 while [[ "$1" != "" ]] ; do
 
-case "$1" in
 
---help|-h)
+function print_usage() {
   echo "Usage: buildall [OPTION]"
   echo "Options:"
   echo "   --profile=DIST_PROFILE   profile for the dist module, default: noSnapshots,"
-  echo "                            also supported: snapshots, minimumFeatureVersionMix"
+  echo "                            also supported: snapshots, minimumFeatureVersionMix, snapshotsWithDataBricks,"
+  echo "                            and noSnapshotsWithDataBricks. NOTE: the Databricks-related spark3XYdb shims"
+  echo "                            are not built locally, the jars are fetched prebuilt from a remote Maven repo."
   echo "   --profile=BUILDVER       builds a single Spark shim, e.g. --profile=320"
   echo "   --module=MODULE          after finishing parallel builds, resume from dist and build"
   echo "                            up to and including module MODULE, e.g., integration_tests"
+  echo "   --parallel=N             build in parallel, N is passed via -P to xargs. NOTE: standard maven does not support"
+  echo "                            concurrent installs. If you see an error related to a maven-metadata-local.xml in"
+  echo "                            your local m2 cache, delete its parent directory and retry."
+}
+
+case "$1" in
+
+--help|-h)
+  print_usage
   exit 0
   ;;
 
 -p=*|--profile=*)
   DIST_PROFILE="${1#*=}"
-  shift
   ;;
 
 -m=*|--module=*)
   MODULE="${1#*=}"
-  shift
+  ;;
+
+-P=*|--parallel=*)
+  BUILD_PARALLEL="${1#*=}"
   ;;
 
 *)
   echo >&2 "Unknown arg: $1"
+  print_usage
   exit 1
   ;;
 
 esac
 
+# advance $1 to the next in the arg list
+shift
+
 done
 
 # TODO create a switch for debug
@@ -59,7 +77,7 @@ DIST_PROFILE=${DIST_PROFILE:-"noSnapshots"}
 
 case $DIST_PROFILE in
 
-  snapshots)
+  snapshots?(WithDatabricks))
     SPARK_SHIM_VERSIONS=(
       301
       302
@@ -73,7 +91,7 @@ case $DIST_PROFILE in
     )
     ;;
 
-  noSnapshots)
+  noSnapshots?(WithDatabricks))
     SPARK_SHIM_VERSIONS=(
       301
       302
@@ -100,7 +118,7 @@ case $DIST_PROFILE in
     ;;
 
   *)
-    echo "ERROR: Supported profiles are: snapshots, noSnapshots, minimumFeatureVersionMix"
+    echo "ERROR unexpected value for profile: $DIST_PROFILE, see 'buildall --help'"
     exit 1
 
 esac
@@ -109,23 +127,24 @@ esac
 
 # First element in SPARK_SHIM_VERSIONS to do most of the checks
 export BASE_VER=${SPARK_SHIM_VERSIONS[0]}
-
-BUILD_PARALLEL=${BUILD_PARALLEL:-4}
+export NUM_SHIMS=${#SPARK_SHIM_VERSIONS[@]}
+export BUILD_PARALLEL=${BUILD_PARALLEL:-4}
 
 echo Clean once across all modules
 mvn -q clean
 
-echo Building a combined dist jar with Shims for "${SPARK_SHIM_VERSIONS[@]}" ...
+echo "Building a combined dist jar with Shims for ${SPARK_SHIM_VERSIONS[@]} ..."
 
 export MVN_BASE_DIR=$(mvn help:evaluate -Dexpression=project.basedir -q -DforceStdout)
 build_single_shim() {
   set -x
   BUILD_VER=$1
   mkdir -p "$MVN_BASE_DIR/target"
-  LOG_FILE="$MVN_BASE_DIR/target/mvn-build-$BUILD_VER.log"
+  (( BUILD_PARALLEL == 1 || NUM_SHIMS == 1 )) && LOG_FILE="/dev/tty" || \
+    LOG_FILE="$MVN_BASE_DIR/target/mvn-build-$BUILD_VER.log"
   SKIP_CHECKS=$( [[ "$BUILD_VER" == "$BASE_VER" ]] && echo false || echo true )
 
-  echo "#### REDIRECTING parallel mvn output to $LOG_FILE ####"
+  echo "#### REDIRECTING mvn output to $LOG_FILE ####"
   mvn -U install \
       -DskipTests \
       -Dbuildver="$BUILD_VER" \
@@ -134,7 +153,7 @@ build_single_shim() {
       -Dskip="$SKIP_CHECKS" \
       -Dmaven.scalastyle.skip="$SKIP_CHECKS" \
       -pl aggregator -am > "$LOG_FILE" 2>&1 || {
-        tail -20 "$LOG_FILE"
+        [[ "$LOG_FILE" != "/dev/tty" ]] && tail -20 "$LOG_FILE" || true
         exit 255
       }
 }
@@ -151,17 +170,13 @@ export -f build_single_shim
 #   regressions even before pytest-based integration_tests
 #
 # Then resume maven build from the dist module now that shims have been installed
-
-# don't regenerate config markdowns unless 301 is included
-SKIP_UPDATE_CONFIG=$( [[ "$BASE_VER" == "301" ]] && echo false || echo true )
 time (
   # printf a single buildver array element per line
   printf "%s\n" "${SPARK_SHIM_VERSIONS[@]}" | \
     xargs -t -I% -P "$BUILD_PARALLEL" -n 1 \
     bash -c 'build_single_shim "$@"' _ %
   echo "Resuming from dist build only using $BASE_VER"
-  mvn -U install --resume-from dist $MODULE_OPT $MVN_PROFILE_OPT \
+  mvn install --resume-from dist $MODULE_OPT $MVN_PROFILE_OPT \
     -Dbuildver="$BASE_VER" \
-    -DskipTests \
-    -Dmaven.updateconfig.skip="$SKIP_UPDATE_CONFIG"
+    -DskipTests
 )

dist/maven-antrun/build-parallel-worlds.xml

@@ -0,0 +1,79 @@
+<?xml version="1.0"?>
+<!--
+  Copyright (c) 2021, NVIDIA CORPORATION.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project name="maven-antrun" xmlns:ac="antlib:net.sf.antcontrib">
+    <!--
+        - Generates shim service discovery META-INF entries for ShimLoader
+
+        - Creates per-spark-build shim root directories (parallel worlds) from versioned aggregators
+
+        - Uses whichever aggregator from included_buildvers earliest in the list can contribute to
+          the conventional class file locations (shared world) without overwriting them. This is just
+          for deterministic retracing what aggregator contributed a particular file. The shared world
+          files should be bitwise-identical as a rough approximation of binary compatibility.
+          If we keep buildvers in a consistent order across all profiles, we should be able to fix many
+          issues in one shot. The only anticipated issue is the violation of the bitwise identity check
+          in the binary-dedupe script below.
+
+        - Verifies that all public classes in the shared world can indeed be safely shared because they
+          are bitwise-identical for all Spark version dependencies
+
+        - binary-dedupe ensures that only one bitwise-identical copy per class is stored in the final jar.
+
+        This ant script emulates compile and resource generation phases such that its output
+        can be passed to the maven-jar-plugin to fit well into conventional maven execution logic
+        after side-stepping it in this script.
+
+        The script is in a dedicated file because we can't use ant refid across different maven-antrun-plugin
+        executions, let alone maven profiles. It should be kept as general as possible w.r.t. to a particular
+        shim mix, such that it can generically be called from any profile.
+
+        TODO Once the caller pom is cleaned up enough it should be possible to have a single invocation of this
+        ant script outside profiles, and profiles will just set different values for input properties. At this
+        point it can be inlined again in pom.
+    -->
+    <taskdef resource="net/sf/antcontrib/antcontrib.properties"/>
+
+    <target name="build-parallel-worlds">
+        <echo>Preparing parallel worlds with params: included_buildvers=${included_buildvers}</echo>
+        <property name="shimServiceRsrc" value="META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider"/>
+        <property name="shimServiceFile" value="${project.build.directory}/extra-resources/${shimServiceRsrc}"/>
+        <property name="aggregatorPrefix"
+            value="${project.build.directory}/deps/rapids-4-spark-aggregator_${scala.binary.version}-${project.version}"/>
+        <truncate file="${shimServiceFile}" create="true" mkdirs="true"/>
+        <ac:for list="${included_buildvers}" param="bv">
+            <sequential>
+                <unzip overwrite="false" src="${aggregatorPrefix}-spark@{bv}.jar"
+                    dest="${project.build.directory}/parallel-world">
+                    <patternset id="shared-world-includes">
+                        <includesfile name="${project.basedir}/unshimmed-common-from-spark301.txt"/>
+                        <includesfile name="${project.basedir}/unshimmed-from-each-spark3xx.txt"/>
+                        <includesfile name="${project.basedir}/unshimmed-spark311.txt"/>
+                    </patternset>
+                </unzip>
+                <unzip src="${aggregatorPrefix}-spark@{bv}.jar"
+                    dest="${project.build.directory}/parallel-world/spark@{bv}">
+                </unzip>
+            </sequential>
+        </ac:for>
+
+        <concat destfile="${shimServiceFile}">
+            <fileset dir="${project.build.directory}/parallel-world" includes="spark*/${shimServiceRsrc}"/>
+        </concat>
+        <exec executable="${project.basedir}/scripts/binary-dedupe.sh" failonerror="true"
+            dir="${project.build.directory}"/>
+    </target>
+</project>