Fix #1791, Fix destination path bug in copy scheduler

littlezhou · Jun 4, 2018 · b1b43b9 · b1b43b9
1 parent 90d68ba
commit b1b43b9
Showing 1 changed file with 5 additions and 51 deletions.
diff --git a/...hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/CopyScheduler.java b/...hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/CopyScheduler.java
@@ -136,7 +136,7 @@ public ScheduleResult onSchedule(ActionInfo actionInfo, LaunchAction action) {
     String srcDir = action.getArgs().get(SyncAction.SRC);
     String path = action.getArgs().get(HdfsAction.FILE_PATH);
     String destDir = action.getArgs().get(SyncAction.DEST);
-    String destPath = path.replace(srcDir, destDir);
+    String destPath = path.replaceFirst(srcDir, destDir);
     // Check again to avoid corner cases
     long did = fileDiffChainMap.get(path).getHead();
     if (did == -1) {
@@ -181,7 +181,7 @@ public ScheduleResult onSchedule(ActionInfo actionInfo, LaunchAction action) {
         action.getArgs().put(HdfsAction.FILE_PATH, destPath);
         // TODO scope check
         String remoteDest = fileDiff.getParameters().get("-dest");
-        action.getArgs().put("-dest", remoteDest.replace(srcDir, destDir));
+        action.getArgs().put("-dest", remoteDest.replaceFirst(srcDir, destDir));
         fileDiff.getParameters().remove("-dest");
         break;
       case METADATA:
@@ -361,7 +361,7 @@ private void baseSync(String srcDir,
     Map<String, FileInfo> srcFileSet = new HashMap<>();
     for (FileInfo fileInfo : srcFiles) {
       // Remove prefix/parent
-      srcFileSet.put(fileInfo.getPath().replace(srcDir, ""), fileInfo);
+      srcFileSet.put(fileInfo.getPath().replaceFirst(srcDir, ""), fileInfo);
     }
     FileStatus[] fileStatuses = null;
     // recursively file lists
@@ -376,7 +376,7 @@ private void baseSync(String srcDir,
         if (srcFileSet.containsKey(destName)) {
           FileInfo fileInfo = srcFileSet.get(destName);
           String src = fileInfo.getPath();
-          String dest = src.replace(srcDir, destDir);
+          String dest = src.replaceFirst(srcDir, destDir);
           baseSyncQueue.put(src, dest);
           srcFileSet.remove(destName);
         }
@@ -389,23 +389,14 @@ private void baseSync(String srcDir,
         continue;
       }
       String src = fileInfo.getPath();
-      String dest = src.replace(srcDir, destDir);
+      String dest = src.replaceFirst(srcDir, destDir);
       baseSyncQueue.put(src, dest);
       overwriteQueue.put(src, true);
       // directSync(src, dest);
     }
     batchDirectSync();
   }
 
-  private void directSync(String src, String srcDir,
-      String destDir) throws MetaStoreException {
-    String dest = src.replace(srcDir, destDir);
-    FileDiff fileDiff = directSync(src, dest);
-    if (fileDiff != null) {
-      metaStore.insertFileDiff(fileDiff);
-    }
-  }
-
   private FileDiff directSync(String src, String dest) throws MetaStoreException {
     FileInfo fileInfo = metaStore.getFile(src);
     if (fileInfo == null) {
@@ -501,17 +492,6 @@ private void addDiffToCache(FileDiff fileDiff) throws MetaStoreException {
     fileDiffCache.put(fileDiff.getDiffId(), fileDiff);
   }
 
-/*  private void updateFileDiffInCache(FileDiff fileDiff) {
-    // judge whether change the file diff
-    if (fileDiffCache.containsKey(fileDiff.getDiffId())) {
-      FileDiff oldDiff = fileDiffCache.get(fileDiff.getDiffId());
-      if (!oldDiff.equals(fileDiff)) {
-        fileDiffCacheChanged.put(fileDiff.getDiffId(), true);
-        fileDiffCache.put(fileDiff.getDiffId(), fileDiff);
-      }
-    }
-  }*/
-
   private void updateFileDiffInCache(Long did,
       FileDiffState fileDiffState) throws MetaStoreException {
     LOG.debug("Update FileDiff");
@@ -594,32 +574,6 @@ public void stop() throws IOException {
     executorService.shutdown();
   }
 
-  private void lockFile(String fileName) {
-    fileLock.put(fileName, 0L);
-  }
-
-  private void lockFile(long did) {
-    FileDiff diff = fileDiffCache.get(did);
-    if(diff == null) {
-      return;
-    }
-    fileLock.put(diff.getSrc(), did);
-  }
-
-  private void unlockFile(String fileName) {
-    if (fileLock.containsKey(fileName)) {
-      fileLock.remove(fileName);
-    }
-  }
-
-  private void unlockFile(long did){
-    FileDiff diff = fileDiffCache.get(did);
-    if(diff == null) {
-      return;
-    }
-    fileLock.remove(diff.getSrc());
-  }
-
   private boolean fileExistOnStandby(String filePath) {
     // TODO Need to be more general to handle failure
     try {