From 52e6995464ef92754a9a020d333960785a2b017c Mon Sep 17 00:00:00 2001 From: ZhenDu <454858191@qq.com> Date: Tue, 8 Aug 2017 11:29:59 +0800 Subject: [PATCH] Fix #892, add src (varchar(4096)) field to file_diff table. (#896) --- .../java/org/smartdata/model/FileDiff.java | 49 +++++++++---------- .../metric/fetcher/InotifyEventApplier.java | 26 +++++----- .../smartdata/metastore/dao/FileDiffDao.java | 2 + .../metastore/utils/MetaStoreUtils.java | 1 + .../metastore/dao/TestFileDiffDao.java | 4 ++ 5 files changed, 44 insertions(+), 38 deletions(-) diff --git a/smart-common/src/main/java/org/smartdata/model/FileDiff.java b/smart-common/src/main/java/org/smartdata/model/FileDiff.java index 23c1f014a47..269334c4199 100644 --- a/smart-common/src/main/java/org/smartdata/model/FileDiff.java +++ b/smart-common/src/main/java/org/smartdata/model/FileDiff.java @@ -20,10 +20,19 @@ public class FileDiff { private long diffId; private String parameters; + private String src; private FileDiffType diffType; private boolean applied; private long create_time; + public String getSrc() { + return src; + } + + public void setSrc(String src) { + this.src = src; + } + public long getDiffId() { return diffId; } @@ -66,44 +75,34 @@ public void setCreate_time(long create_time) { @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof FileDiff)) { - return false; - } + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; FileDiff fileDiff = (FileDiff) o; - if (getDiffId() != fileDiff.getDiffId()) { - return false; - } - if (isApplied() != fileDiff.isApplied()) { - return false; - } - if (getCreate_time() != fileDiff.getCreate_time()) { - return false; - } - if (!getParameters().equals(fileDiff.getParameters())) { - return false; - } - return getDiffType() == fileDiff.getDiffType(); + if (diffId != fileDiff.diffId) return false; + if (applied != fileDiff.applied) return false; + if (create_time != fileDiff.create_time) return false; + if (parameters != null ? !parameters.equals(fileDiff.parameters) : fileDiff.parameters != null) return false; + if (src != null ? !src.equals(fileDiff.src) : fileDiff.src != null) return false; + return diffType == fileDiff.diffType; } @Override public int hashCode() { - int result = (int) (getDiffId() ^ (getDiffId() >>> 32)); - result = 31 * result + getParameters().hashCode(); - result = 31 * result + getDiffType().hashCode(); - result = 31 * result + (isApplied() ? 1 : 0); - result = 31 * result + (int) (getCreate_time() ^ (getCreate_time() >>> 32)); + int result = (int) (diffId ^ (diffId >>> 32)); + result = 31 * result + (parameters != null ? parameters.hashCode() : 0); + result = 31 * result + (src != null ? src.hashCode() : 0); + result = 31 * result + (diffType != null ? diffType.hashCode() : 0); + result = 31 * result + (applied ? 1 : 0); + result = 31 * result + (int) (create_time ^ (create_time >>> 32)); return result; } @Override public String toString() { return String.format("FileDiff{diffId=%s, parameters=%s, " + - "diffType=%s, applied=%s, create_time=%s}", diffId, parameters, + "src=%s, diffType=%s, applied=%s, create_time=%s}", diffId, parameters, src, diffType, applied, create_time); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java index ba73915d7ae..28015633725 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java @@ -77,28 +77,28 @@ private List getSqlStatement(Event event) throws IOException, MetaStoreE // TODO parse and save to fileDiff case CREATE: LOG.trace("event type:" + event.getEventType().name() + - ", path:" + ((Event.CreateEvent)event).getPath()); + ", path:" + ((Event.CreateEvent) event).getPath()); /*fileDiff.setDiffType(FileDiffType.CREATE); fileDiff.setParameters(String.format("-file %s", ((Event.CreateEvent)event).getPath())); metaStore.insertFileDiff(fileDiff);*/ - return Arrays.asList(this.getCreateSql((Event.CreateEvent)event)); + return Arrays.asList(this.getCreateSql((Event.CreateEvent) event)); case CLOSE: LOG.trace("event type:" + event.getEventType().name() + - ", path:" + ((Event.CloseEvent)event).getPath()); + ", path:" + ((Event.CloseEvent) event).getPath()); fileDiff.setDiffType(FileDiffType.APPEND); - fileDiff.setParameters(String.format("-file %s -length %s", - ((Event.CloseEvent)event).getPath(), - ((Event.CloseEvent)event).getFileSize())); + fileDiff.setSrc(String.format("%s", ((Event.CloseEvent) event).getPath())); + fileDiff.setParameters(String.format("-length %s", + ((Event.CloseEvent) event).getFileSize())); metaStore.insertFileDiff(fileDiff); - return Arrays.asList(this.getCloseSql((Event.CloseEvent)event)); + return Arrays.asList(this.getCloseSql((Event.CloseEvent) event)); case RENAME: LOG.trace("event type:" + event.getEventType().name() + - ", src path:" + ((Event.RenameEvent)event).getSrcPath() + - ", dest path:" + ((Event.RenameEvent)event).getDstPath()); + ", src path:" + ((Event.RenameEvent) event).getSrcPath() + + ", dest path:" + ((Event.RenameEvent) event).getDstPath()); fileDiff.setDiffType(FileDiffType.RENAME); - fileDiff.setParameters(String.format("-file %s -dest %s", - ((Event.RenameEvent)event).getSrcPath(), + fileDiff.setSrc(String.format("%s",((Event.RenameEvent)event).getSrcPath())); + fileDiff.setParameters(String.format("-dest %s", ((Event.RenameEvent)event).getDstPath())); metaStore.insertFileDiff(fileDiff); return this.getRenameSql((Event.RenameEvent)event); @@ -114,8 +114,8 @@ private List getSqlStatement(Event event) throws IOException, MetaStoreE LOG.trace("event type:" + event.getEventType().name() + ", path:" + ((Event.UnlinkEvent)event).getPath()); fileDiff.setDiffType(FileDiffType.DELETE); - fileDiff.setParameters(String.format("-file %s", - ((Event.UnlinkEvent)event).getPath())); + fileDiff.setSrc(String.format("%s",((Event.UnlinkEvent)event).getPath())); + fileDiff.setParameters(""); metaStore.insertFileDiff(fileDiff); return this.getUnlinkSql((Event.UnlinkEvent)event); } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileDiffDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileDiffDao.java index 093c53e50c9..67a3bf05874 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileDiffDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileDiffDao.java @@ -112,6 +112,7 @@ private Map toMap(FileDiff fileDiff) { Map parameters = new HashMap<>(); parameters.put("did", fileDiff.getDiffId()); parameters.put("diff_type", fileDiff.getDiffType().getValue()); + parameters.put("src", fileDiff.getSrc()); parameters.put("parameters", fileDiff.getParameters()); parameters.put("applied", fileDiff.isApplied()); parameters.put("create_time", fileDiff.getCreate_time()); @@ -127,6 +128,7 @@ public FileDiff mapRow(ResultSet resultSet, int i) throws SQLException { fileDiff.setApplied(resultSet.getBoolean("applied")); fileDiff.setCreate_time(resultSet.getLong("create_time")); fileDiff.setParameters(resultSet.getString("parameters")); + fileDiff.setSrc(resultSet.getString("src")); return fileDiff; } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java b/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java index 57ca2a2e898..bd7d656fa6d 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java @@ -240,6 +240,7 @@ public static void initializeDataBase( "CREATE TABLE file_diff (\n" + " did INTEGER PRIMARY KEY AUTOINCREMENT,\n" + " diff_type varchar(4096) NOT NULL,\n" + + " src varchar(4096) NOT NULL,\n" + " parameters varchar(4096) NOT NULL,\n" + " applied tinyint(4) NOT NULL,\n" + " create_time bigint(20) NOT NULL\n" + diff --git a/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestFileDiffDao.java b/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestFileDiffDao.java index 2c45764ac36..fd4755eb9c9 100644 --- a/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestFileDiffDao.java +++ b/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestFileDiffDao.java @@ -48,6 +48,7 @@ public void testInsertAndGetSingleRecord() { FileDiff fileDiff = new FileDiff(); fileDiff.setDiffId(1); fileDiff.setParameters("test"); + fileDiff.setSrc("test"); fileDiff.setApplied(true); fileDiff.setDiffType(FileDiffType.APPEND); fileDiff.setCreate_time(1); @@ -61,6 +62,7 @@ public void testBatchInsertAndQuery() { fileDiffs[0] = new FileDiff(); fileDiffs[0].setDiffId(1); fileDiffs[0].setParameters("test"); + fileDiffs[0].setSrc("test"); fileDiffs[0].setApplied(true); fileDiffs[0].setDiffType(FileDiffType.APPEND); fileDiffs[0].setCreate_time(1); @@ -68,6 +70,7 @@ public void testBatchInsertAndQuery() { fileDiffs[1] = new FileDiff(); fileDiffs[1].setDiffId(2); fileDiffs[1].setParameters("test"); + fileDiffs[1].setSrc("test"); fileDiffs[1].setApplied(true); fileDiffs[1].setDiffType(FileDiffType.APPEND); fileDiffs[1].setCreate_time(1); @@ -84,6 +87,7 @@ public void testUpdate() { FileDiff fileDiff = new FileDiff(); fileDiff.setDiffId(1); fileDiff.setParameters("test"); + fileDiff.setSrc("test"); fileDiff.setApplied(true); fileDiff.setDiffType(FileDiffType.APPEND); fileDiff.setCreate_time(1);