Fix #1546, Testing feature: Skip fetch entire HDFS namespace and upda…

…te based on iNotify only (#1547)
littlezhou · Jan 30, 2018 · 1082693 · 1082693
1 parent 14472d6
commit 1082693
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 6 deletions.
diff --git a/conf/smart-default.xml b/conf/smart-default.xml
@@ -137,4 +137,13 @@
     <value>7070</value>
     <description>SmartAgent master port</description>
   </property>
+
+  <property>
+    <name>smart.namespace.fetcher.ignore.unsuccessive.inotify.event</name>
+    <value>false</value>
+    <description>
+      Skip fetch the entire namespace and only use available iNotify events to update namespace if true.
+      NOTE: This may leads to some unpredictable consequences and should only be used for testing.
+    </description>
+  </property>
 </configuration>
diff --git a/smart-common/src/main/java/org/smartdata/conf/SmartConfKeys.java b/smart-common/src/main/java/org/smartdata/conf/SmartConfKeys.java
@@ -37,11 +37,17 @@ public class SmartConfKeys {
 
   public static final String SMART_DFS_NAMENODE_RPCSERVER_KEY = "smart.dfs.namenode.rpcserver";
 
-  // confKeys for alluxio
+  // Configure keys for HDFS
+  public static final String SMART_NAMESPACE_FETCHER_IGNORE_UNSUCCESSIVE_INOTIFY_EVENT_KEY =
+      "smart.namespace.fetcher.ignore.unsuccessive.inotify.event";
+  public static final boolean SMART_NAMESPACE_FETCHER_IGNORE_UNSUCCESSIVE_INOTIFY_EVENT_DEFAULT =
+      false;
+
+  // Configure keys for Alluxio
   public static final String SMART_ALLUXIO_MASTER_HOSTNAME_KEY = "smart.alluxio.master.hostname";
   public static final String SMART_ALLUXIO_CONF_DIR_KEY = "smart.alluxio.conf.dir";
 
-  //ssm
+  // SSM
   public static final String SMART_SERVER_RPC_ADDRESS_KEY = "smart.server.rpc.address";
   public static final String SMART_SERVER_RPC_ADDRESS_DEFAULT = "0.0.0.0:7042";
   public static final String SMART_SERVER_RPC_HANDLER_COUNT_KEY = "smart.server.rpc.handler.count";

diff --git a/...ort/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventFetcher.java b/...ort/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventFetcher.java
@@ -102,11 +102,21 @@ public InotifyEventFetcher(DFSClient client, MetaStore metaStore,
   }
 
   public void start() throws IOException {
-    Long lastTxid = getLastTxid();
-    if (lastTxid != null && lastTxid != -1 && canContinueFromLastTxid(client, lastTxid)) {
-      startFromLastTxid(lastTxid);
+    boolean ignore = conf.getBoolean(
+        SmartConfKeys.SMART_NAMESPACE_FETCHER_IGNORE_UNSUCCESSIVE_INOTIFY_EVENT_KEY,
+        SmartConfKeys.SMART_NAMESPACE_FETCHER_IGNORE_UNSUCCESSIVE_INOTIFY_EVENT_DEFAULT);
+    if (!ignore) {
+      Long lastTxid = getLastTxid();
+      if (lastTxid != null && lastTxid != -1 && canContinueFromLastTxid(client, lastTxid)) {
+        startFromLastTxid(lastTxid);
+      } else {
+        startWithFetchingNameSpace();
+      }
     } else {
-      startWithFetchingNameSpace();
+      long id = client.getNamenode().getCurrentEditLogTxid();
+      LOG.warn("NOTE: Incomplete iNotify event may cause unpredictable consequences. "
+          + "This should only be used for testing.");
+      startFromLastTxid(id);
     }
   }