Fix #892, add src (varchar(4096)) field to file_diff table. (#896)

Intel-bigdata · Aug 8, 2017 · 52e6995 · 52e6995
1 parent abe8212
commit 52e6995
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 38 deletions.
diff --git a/smart-common/src/main/java/org/smartdata/model/FileDiff.java b/smart-common/src/main/java/org/smartdata/model/FileDiff.java
@@ -20,10 +20,19 @@
 public class FileDiff {
   private long diffId;
   private String parameters;
+  private String src;
   private FileDiffType diffType;
   private boolean applied;
   private long create_time;
 
+  public String getSrc() {
+    return src;
+  }
+
+  public void setSrc(String src) {
+    this.src = src;
+  }
+
   public long getDiffId() {
     return diffId;
   }
@@ -66,44 +75,34 @@ public void setCreate_time(long create_time) {
 
   @Override
   public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (!(o instanceof FileDiff)) {
-      return false;
-    }
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
 
     FileDiff fileDiff = (FileDiff) o;
 
-    if (getDiffId() != fileDiff.getDiffId()) {
-      return false;
-    }
-    if (isApplied() != fileDiff.isApplied()) {
-      return false;
-    }
-    if (getCreate_time() != fileDiff.getCreate_time()) {
-      return false;
-    }
-    if (!getParameters().equals(fileDiff.getParameters())) {
-      return false;
-    }
-    return getDiffType() == fileDiff.getDiffType();
+    if (diffId != fileDiff.diffId) return false;
+    if (applied != fileDiff.applied) return false;
+    if (create_time != fileDiff.create_time) return false;
+    if (parameters != null ? !parameters.equals(fileDiff.parameters) : fileDiff.parameters != null) return false;
+    if (src != null ? !src.equals(fileDiff.src) : fileDiff.src != null) return false;
+    return diffType == fileDiff.diffType;
   }
 
   @Override
   public int hashCode() {
-    int result = (int) (getDiffId() ^ (getDiffId() >>> 32));
-    result = 31 * result + getParameters().hashCode();
-    result = 31 * result + getDiffType().hashCode();
-    result = 31 * result + (isApplied() ? 1 : 0);
-    result = 31 * result + (int) (getCreate_time() ^ (getCreate_time() >>> 32));
+    int result = (int) (diffId ^ (diffId >>> 32));
+    result = 31 * result + (parameters != null ? parameters.hashCode() : 0);
+    result = 31 * result + (src != null ? src.hashCode() : 0);
+    result = 31 * result + (diffType != null ? diffType.hashCode() : 0);
+    result = 31 * result + (applied ? 1 : 0);
+    result = 31 * result + (int) (create_time ^ (create_time >>> 32));
     return result;
   }
 
   @Override
   public String toString() {
     return String.format("FileDiff{diffId=%s, parameters=%s, " +
-            "diffType=%s, applied=%s, create_time=%s}", diffId, parameters,
+            "src=%s, diffType=%s, applied=%s, create_time=%s}", diffId, parameters, src,
         diffType, applied, create_time);
   }
 }
diff --git a/...ort/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java b/...ort/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java
@@ -77,28 +77,28 @@ private List<String> getSqlStatement(Event event) throws IOException, MetaStoreE
       // TODO parse and save to fileDiff
       case CREATE:
         LOG.trace("event type:" + event.getEventType().name() +
-            ", path:" + ((Event.CreateEvent)event).getPath());
+            ", path:" + ((Event.CreateEvent) event).getPath());
         /*fileDiff.setDiffType(FileDiffType.CREATE);
         fileDiff.setParameters(String.format("-file %s",
             ((Event.CreateEvent)event).getPath()));
         metaStore.insertFileDiff(fileDiff);*/
-        return Arrays.asList(this.getCreateSql((Event.CreateEvent)event));
+        return Arrays.asList(this.getCreateSql((Event.CreateEvent) event));
       case CLOSE:
         LOG.trace("event type:" + event.getEventType().name() +
-            ", path:" + ((Event.CloseEvent)event).getPath());
+            ", path:" + ((Event.CloseEvent) event).getPath());
         fileDiff.setDiffType(FileDiffType.APPEND);
-        fileDiff.setParameters(String.format("-file %s -length %s",
-            ((Event.CloseEvent)event).getPath(),
-            ((Event.CloseEvent)event).getFileSize()));
+        fileDiff.setSrc(String.format("%s", ((Event.CloseEvent) event).getPath()));
+        fileDiff.setParameters(String.format("-length %s",
+            ((Event.CloseEvent) event).getFileSize()));
         metaStore.insertFileDiff(fileDiff);
-        return Arrays.asList(this.getCloseSql((Event.CloseEvent)event));
+        return Arrays.asList(this.getCloseSql((Event.CloseEvent) event));
       case RENAME:
         LOG.trace("event type:" + event.getEventType().name() +
-            ", src path:" + ((Event.RenameEvent)event).getSrcPath() +
-            ", dest path:" + ((Event.RenameEvent)event).getDstPath());
+            ", src path:" + ((Event.RenameEvent) event).getSrcPath() +
+            ", dest path:" + ((Event.RenameEvent) event).getDstPath());
         fileDiff.setDiffType(FileDiffType.RENAME);
-        fileDiff.setParameters(String.format("-file %s -dest %s",
-            ((Event.RenameEvent)event).getSrcPath(),
+        fileDiff.setSrc(String.format("%s",((Event.RenameEvent)event).getSrcPath()));
+        fileDiff.setParameters(String.format("-dest %s",
             ((Event.RenameEvent)event).getDstPath()));
         metaStore.insertFileDiff(fileDiff);
         return this.getRenameSql((Event.RenameEvent)event);
@@ -114,8 +114,8 @@ private List<String> getSqlStatement(Event event) throws IOException, MetaStoreE
         LOG.trace("event type:" + event.getEventType().name() +
             ", path:" + ((Event.UnlinkEvent)event).getPath());
         fileDiff.setDiffType(FileDiffType.DELETE);
-        fileDiff.setParameters(String.format("-file %s",
-            ((Event.UnlinkEvent)event).getPath()));
+        fileDiff.setSrc(String.format("%s",((Event.UnlinkEvent)event).getPath()));
+        fileDiff.setParameters("");
         metaStore.insertFileDiff(fileDiff);
         return this.getUnlinkSql((Event.UnlinkEvent)event);
     }

diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileDiffDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileDiffDao.java
@@ -112,6 +112,7 @@ private Map<String, Object> toMap(FileDiff fileDiff) {
     Map<String, Object> parameters = new HashMap<>();
     parameters.put("did", fileDiff.getDiffId());
     parameters.put("diff_type", fileDiff.getDiffType().getValue());
+    parameters.put("src", fileDiff.getSrc());
     parameters.put("parameters", fileDiff.getParameters());
     parameters.put("applied", fileDiff.isApplied());
     parameters.put("create_time", fileDiff.getCreate_time());
@@ -127,6 +128,7 @@ public FileDiff mapRow(ResultSet resultSet, int i) throws SQLException {
       fileDiff.setApplied(resultSet.getBoolean("applied"));
       fileDiff.setCreate_time(resultSet.getLong("create_time"));
       fileDiff.setParameters(resultSet.getString("parameters"));
+      fileDiff.setSrc(resultSet.getString("src"));
 
       return fileDiff;
     }

diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java b/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java
@@ -240,6 +240,7 @@ public static void initializeDataBase(
         "CREATE TABLE file_diff (\n" +
             "  did INTEGER PRIMARY KEY AUTOINCREMENT,\n" +
             "  diff_type varchar(4096) NOT NULL,\n" +
+            "  src varchar(4096) NOT NULL,\n" +
             "  parameters varchar(4096) NOT NULL,\n" +
             "  applied tinyint(4) NOT NULL,\n" +
             "  create_time bigint(20) NOT NULL\n" +

diff --git a/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestFileDiffDao.java b/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestFileDiffDao.java
@@ -48,6 +48,7 @@ public void testInsertAndGetSingleRecord() {
     FileDiff fileDiff = new FileDiff();
     fileDiff.setDiffId(1);
     fileDiff.setParameters("test");
+    fileDiff.setSrc("test");
     fileDiff.setApplied(true);
     fileDiff.setDiffType(FileDiffType.APPEND);
     fileDiff.setCreate_time(1);
@@ -61,13 +62,15 @@ public void testBatchInsertAndQuery() {
     fileDiffs[0] = new FileDiff();
     fileDiffs[0].setDiffId(1);
     fileDiffs[0].setParameters("test");
+    fileDiffs[0].setSrc("test");
     fileDiffs[0].setApplied(true);
     fileDiffs[0].setDiffType(FileDiffType.APPEND);
     fileDiffs[0].setCreate_time(1);
 
     fileDiffs[1] = new FileDiff();
     fileDiffs[1].setDiffId(2);
     fileDiffs[1].setParameters("test");
+    fileDiffs[1].setSrc("test");
     fileDiffs[1].setApplied(true);
     fileDiffs[1].setDiffType(FileDiffType.APPEND);
     fileDiffs[1].setCreate_time(1);
@@ -84,6 +87,7 @@ public void testUpdate() {
     FileDiff fileDiff = new FileDiff();
     fileDiff.setDiffId(1);
     fileDiff.setParameters("test");
+    fileDiff.setSrc("test");
     fileDiff.setApplied(true);
     fileDiff.setDiffType(FileDiffType.APPEND);
     fileDiff.setCreate_time(1);