revert unnecessary configs

LiveIntent · Oct 25, 2023 · 217577c · 217577c
1 parent f0e3f4d
commit 217577c
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 29 deletions.
diff --git a/clients/venice-push-job/src/main/java/com/linkedin/venice/hadoop/VenicePushJob.java b/clients/venice-push-job/src/main/java/com/linkedin/venice/hadoop/VenicePushJob.java
@@ -347,7 +347,7 @@ public class VenicePushJob implements AutoCloseable {
   public static final String VALUE_SCHEMA_ID_PROP = "value.schema.id";
   public static final String DERIVED_SCHEMA_ID_PROP = "derived.schema.id";
   public static final String TOPIC_PROP = "venice.kafka.topic";
-  public static final String HADOOP_PREFIX = "hadoop-conf.";
+  protected static final String HADOOP_PREFIX = "hadoop-conf.";
   protected static final String HADOOP_VALIDATE_SCHEMA_AND_BUILD_DICT_PREFIX = "hadoop-dict-build-conf.";
   public static final String SSL_PREFIX = "ssl";
 
@@ -948,7 +948,6 @@ public void run() {
           "The store {} is discovered in Venice cluster {}",
           pushJobSetting.storeName,
           pushJobSetting.clusterName);
-      HadoopUtils.setHadoopConfigurationFromProperties(jobConf, props);
 
       if (pushJobSetting.isSourceKafka) {
         initKIFRepushDetails();
@@ -3081,6 +3080,10 @@ protected void setupDefaultJobConf(
 
     for (String key: props.keySet()) {
       String lowerCase = key.toLowerCase();
+      if (lowerCase.startsWith(HADOOP_PREFIX)) {
+        String overrideKey = key.substring(HADOOP_PREFIX.length());
+        conf.set(overrideKey, props.getString(key));
+      }
       for (String prefix: passThroughPrefixList) {
         if (lowerCase.startsWith(prefix)) {
           conf.set(key, props.getString(key));

diff --git a/clients/venice-push-job/src/main/java/com/linkedin/venice/hadoop/utils/HadoopUtils.java b/clients/venice-push-job/src/main/java/com/linkedin/venice/hadoop/utils/HadoopUtils.java
@@ -1,7 +1,5 @@
 package com.linkedin.venice.hadoop.utils;
 
-import static com.linkedin.venice.hadoop.VenicePushJob.HADOOP_PREFIX;
-
 import com.linkedin.venice.utils.VeniceProperties;
 import java.io.IOException;
 import java.util.Properties;
@@ -54,17 +52,6 @@ public static void cleanUpHDFSPath(String path, boolean recursive) {
     }
   }
 
-  public static void setHadoopConfigurationFromProperties(Configuration conf, VeniceProperties props) {
-    for (String key: props.keySet()) {
-      String lowerCase = key.toLowerCase();
-      if (lowerCase.startsWith(HADOOP_PREFIX)) {
-        String overrideKey = key.substring(HADOOP_PREFIX.length());
-        conf.set(overrideKey, props.getString(key));
-        LOGGER.info("Hadoop configuration {} is overwritten by {}", overrideKey, key);
-      }
-    }
-  }
-
   private HadoopUtils() {
   }
 }
diff --git a/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/utils/TestHadoopUtils.java b/clients/venice-push-job/src/test/java/com/linkedin/venice/hadoop/utils/TestHadoopUtils.java
@@ -1,8 +1,6 @@
 package com.linkedin.venice.hadoop.utils;
 
-import com.linkedin.venice.utils.VeniceProperties;
 import java.io.IOException;
-import java.util.Properties;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -42,16 +40,4 @@ public void testCleanUpHDFSPath() throws IOException {
     // validate the path
     Assert.assertFalse(fs.exists(p));
   }
-
-  @Test
-  public void testSetHadoopConfigurationFromProperties() {
-    Configuration conf = new Configuration();
-    Properties innerProps = new Properties();
-    innerProps.setProperty("non.valid.key", "shouldn't exist");
-    innerProps.setProperty("hadoop-conf.fs.s3a.access.key", "s3-key");
-    VeniceProperties props = new VeniceProperties(innerProps);
-    HadoopUtils.setHadoopConfigurationFromProperties(conf, props);
-    Assert.assertEquals(conf.get("fs.s3a.access.key"), "s3-key");
-    Assert.assertNull(conf.get("non.valid.key"));
-  }
 }