bump spark and hadoop

build.gradle

@@ -49,8 +49,8 @@ def jacksonVersion = '2.13.4'
 def pulsarGroup = 'org.apache.pulsar'
 def pulsarVersion = '2.10.3'
 def alpnAgentVersion = '2.0.10'
-def hadoopVersion = '2.10.2'
-def apacheSparkVersion = '3.3.3'
+def hadoopVersion = '3.4.0'
+def apacheSparkVersion = '3.4.3'
 def antlrVersion = '4.8'
 
 ext.libraries = [

clients/venice-push-job/build.gradle

@@ -21,13 +21,13 @@ dependencies {
     exclude group: 'javax.servlet'
   }
 
-  implementation (libraries.hadoopCommon) {
+  compileOnly (libraries.hadoopCommon) {
     // Exclude transitive dependency
     exclude group: 'org.apache.avro'
     exclude group: 'javax.servlet'
   }
 
-  implementation (libraries.hadoopHdfs) {
+  compileOnly (libraries.hadoopHdfs) {
     // Exclude transitive dependency
     exclude group: 'org.apache.avro'
     exclude group: 'javax.servlet'
@@ -49,7 +49,7 @@ dependencies {
     // Spark 3.3 depends on log4j 2.17.2 which has a performance regression (LOG4J2-3487)
     exclude group: 'org.apache.logging.log4j'
   }
-  implementation (libraries.apacheSparkCore) {
+  compileOnly (libraries.apacheSparkCore) {
     // Spark 3.1 depends on Avro 1.8.2 - which uses avro-mapred with the hadoop2 classifier. Starting from Avro 1.9
     // onwards, avro-mapred is no longer published with a hadoop2 classifier, but Gradle still looks for one.
     exclude group: 'org.apache.avro', module: 'avro-mapred'
@@ -64,7 +64,7 @@ dependencies {
     // Spark 3.3 depends on log4j 2.17.2 which has a performance regression (LOG4J2-3487)
     exclude group: 'org.apache.logging.log4j'
   }
-  implementation (libraries.apacheSparkSql) {
+  compileOnly (libraries.apacheSparkSql) {
     // Spark 3.1 depends on Avro 1.8.2 - which uses avro-mapred with the hadoop2 classifier. Starting from Avro 1.9
     // onwards, avro-mapred is no longer published with a hadoop2 classifier, but Gradle still looks for one.
     exclude group: 'org.apache.avro', module: 'avro-mapred'