From bc34b07bd1f029b373ce86e4eeb9eabeb6cef752 Mon Sep 17 00:00:00 2001 From: Qiyuan Gong Date: Sat, 28 Apr 2018 13:57:20 +0800 Subject: [PATCH 1/3] Fix unlink inotify problem --- .../metric/fetcher/InotifyEventApplier.java | 56 +++++++++++++++---- .../fetcher/TestInotifyEventApplier.java | 7 ++- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java index bf2e47cb484..179d4011565 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java @@ -301,17 +301,53 @@ private List getAppendSql(Event.AppendEvent appendEvent) { } private List getUnlinkSql(Event.UnlinkEvent unlinkEvent) throws MetaStoreException { - List fileInfos = metaStore.getFilesByPrefix(unlinkEvent.getPath()); - for (FileInfo fileInfo:fileInfos) { - if (fileInfo.isdir()) { - continue; - } - if (inBackup(unlinkEvent.getPath())) { - FileDiff fileDiff = new FileDiff(FileDiffType.DELETE); - fileDiff.setSrc(unlinkEvent.getPath()); - metaStore.insertFileDiff(fileDiff); + // delete root, i.e., / + String root = "/"; + if (root.equals(unlinkEvent.getPath())) { + LOG.warn("Deleting root directory!!!"); + insertDeleteDiff(root, true); + return Arrays.asList( + String.format("DELETE FROM file WHERE path = '%s%%'", root)); + } + FileInfo fileInfo = metaStore.getFile(unlinkEvent.getPath()); + if (fileInfo == null) return Arrays.asList(); + if (fileInfo.isdir()) { + insertDeleteDiff(unlinkEvent.getPath(), true); + // delete all files in this dir from file table + return Arrays.asList( + String.format("DELETE FROM file WHERE path LIKE '%s%%';", unlinkEvent.getPath())); + } else { + insertDeleteDiff(unlinkEvent.getPath(), false); + // delete file in file table + return Arrays.asList( + String.format("DELETE FROM file WHERE path = '%s';", unlinkEvent.getPath())); + } + } + + private void insertDeleteDiff(String path, boolean isDir) throws MetaStoreException { + if (isDir) { + List fileInfos = metaStore.getFilesByPrefix(path); + for (FileInfo fileInfo : fileInfos) { + // recursively on dir + if (fileInfo.isdir()) { + if (path.equals(fileInfo.getPath())) { + continue; + } + insertDeleteDiff(fileInfo.getPath(), true); + continue; + } + insertDeleteDiff(fileInfo.getPath()); } + } else { + insertDeleteDiff(path); + } + } + + private void insertDeleteDiff(String path) throws MetaStoreException { + if (inBackup(path)) { + FileDiff fileDiff = new FileDiff(FileDiffType.DELETE); + fileDiff.setSrc(path); + metaStore.insertFileDiff(fileDiff); } - return Arrays.asList(String.format("DELETE FROM file WHERE path LIKE '%s%%';", unlinkEvent.getPath())); } } diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java index e2ecd89913f..1dcb130aed0 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java @@ -165,10 +165,11 @@ public void testApplier() throws Exception { Event unlink = new Event.UnlinkEvent.Builder().path("/").timestamp(6).build(); applier.apply(new Event[]{unlink}); - Assert.assertFalse(metaStore.getFile().size() > 0); - + // Assert.assertFalse(metaStore.getFile().size() > 0); + Thread.sleep(3000); + System.out.println("Files in table " + metaStore.getFile().size()); List fileDiffList = metaStore.getPendingDiff(); - Assert.assertTrue(fileDiffList.size() == 3); + Assert.assertTrue(fileDiffList.size() == 4); } @Test From 4d189e11c285f0d98d45eeaeff93f70da3301579 Mon Sep 17 00:00:00 2001 From: Qiyuan Gong Date: Sat, 28 Apr 2018 16:25:33 +0800 Subject: [PATCH 2/3] Fix an issue for dir, fix UT --- .../smartdata/hdfs/metric/fetcher/InotifyEventApplier.java | 5 +++-- .../hdfs/metric/fetcher/TestInotifyEventApplier.java | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java index 179d4011565..4413e68f4a0 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java @@ -307,7 +307,7 @@ private List getUnlinkSql(Event.UnlinkEvent unlinkEvent) throws MetaStor LOG.warn("Deleting root directory!!!"); insertDeleteDiff(root, true); return Arrays.asList( - String.format("DELETE FROM file WHERE path = '%s%%'", root)); + String.format("DELETE FROM file WHERE path like '%s%%'", root)); } FileInfo fileInfo = metaStore.getFile(unlinkEvent.getPath()); if (fileInfo == null) return Arrays.asList(); @@ -315,7 +315,8 @@ private List getUnlinkSql(Event.UnlinkEvent unlinkEvent) throws MetaStor insertDeleteDiff(unlinkEvent.getPath(), true); // delete all files in this dir from file table return Arrays.asList( - String.format("DELETE FROM file WHERE path LIKE '%s%%';", unlinkEvent.getPath())); + String.format("DELETE FROM file WHERE path LIKE '%s/%%';", unlinkEvent.getPath()), + String.format("DELETE FROM file WHERE path = '%s';", unlinkEvent.getPath())); } else { insertDeleteDiff(unlinkEvent.getPath(), false); // delete file in file table diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java index 1dcb130aed0..834cf9c8e19 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java @@ -165,8 +165,8 @@ public void testApplier() throws Exception { Event unlink = new Event.UnlinkEvent.Builder().path("/").timestamp(6).build(); applier.apply(new Event[]{unlink}); - // Assert.assertFalse(metaStore.getFile().size() > 0); - Thread.sleep(3000); + Thread.sleep(1200); + Assert.assertFalse(metaStore.getFile().size() == 0); System.out.println("Files in table " + metaStore.getFile().size()); List fileDiffList = metaStore.getPendingDiff(); Assert.assertTrue(fileDiffList.size() == 4); From 83d4fc647d5c72deaae3b5084e43ea9a7dfe8902 Mon Sep 17 00:00:00 2001 From: Qiyuan Gong Date: Wed, 2 May 2018 12:21:46 +0800 Subject: [PATCH 3/3] Fix testApplier --- .../smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java index 834cf9c8e19..e67c35e83d5 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java @@ -166,7 +166,7 @@ public void testApplier() throws Exception { Event unlink = new Event.UnlinkEvent.Builder().path("/").timestamp(6).build(); applier.apply(new Event[]{unlink}); Thread.sleep(1200); - Assert.assertFalse(metaStore.getFile().size() == 0); + Assert.assertEquals(metaStore.getFile().size(), 0); System.out.println("Files in table " + metaStore.getFile().size()); List fileDiffList = metaStore.getPendingDiff(); Assert.assertTrue(fileDiffList.size() == 4);