Use proto hash and remove hash from db

Signed-off-by: Terence <terencelimxp@gmail.com>
feast-dev · Nov 2, 2020 · cd50a3f · cd50a3f
1 parent ccf4e9a
commit cd50a3f
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 35 deletions.
diff --git a/core/src/main/java/feast/core/model/FeatureTable.java b/core/src/main/java/feast/core/model/FeatureTable.java
@@ -16,8 +16,7 @@
  */
 package feast.core.model;
 
-import com.google.common.primitives.Longs;
-import com.google.protobuf.ByteString;
+import com.google.common.hash.Hashing;
 import com.google.protobuf.Duration;
 import com.google.protobuf.Timestamp;
 import feast.core.dao.EntityRepository;
@@ -26,12 +25,7 @@
 import feast.proto.core.FeatureProto.FeatureSpecV2;
 import feast.proto.core.FeatureTableProto;
 import feast.proto.core.FeatureTableProto.FeatureTableSpec;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
+import java.util.*;
 import java.util.stream.Collectors;
 import javax.persistence.CascadeType;
 import javax.persistence.Column;
@@ -112,9 +106,6 @@ public class FeatureTable extends AbstractTimestampEntity {
   @Column(name = "is_deleted", nullable = false)
   private boolean isDeleted;
 
-  @Column(name = "metadata_hash", nullable = false)
-  private long metadataHash;
-
   public FeatureTable() {};
 
   /**
@@ -155,13 +146,6 @@ public static FeatureTable fromProto(
       table.setStreamSource(DataSource.fromProto(spec.getStreamSource()));
     }
 
-    table.setMetadataHash(
-        Objects.hash(
-            table.getEntities(),
-            table.getFeatures(),
-            table.getBatchSource(),
-            table.getStreamSource()));
-
     return table;
   }
 
@@ -217,11 +201,6 @@ public void updateFromProto(
       this.streamSource = null;
     }
 
-    // Update hash
-    this.setMetadataHash(
-        Objects.hash(
-            this.getEntities(), this.getFeatures(), this.getBatchSource(), this.getStreamSource()));
-
     // Set isDeleted to false
     this.setDeleted(false);
 
@@ -234,14 +213,7 @@ public FeatureTableProto.FeatureTable toProto() {
     // Convert field types to Protobuf compatible types
     Timestamp creationTime = TypeConversion.convertTimestamp(getCreated());
     Timestamp updatedTime = TypeConversion.convertTimestamp(getLastUpdated());
-    ByteString metadataHashBytes =
-        ByteString.copyFrom(
-            Longs.toByteArray(
-                Objects.hash(
-                    this.getEntities(),
-                    this.getFeatures(),
-                    this.getBatchSource(),
-                    this.getStreamSource())));
+    String metadataHashBytes = this.protoHash();
 
     List<FeatureSpecV2> featureSpecs =
         getFeatures().stream().map(FeatureV2::toProto).collect(Collectors.toList());
@@ -316,6 +288,32 @@ public void delete() {
     this.setRevision(0);
   }
 
+  public String protoHash() {
+    List<String> sortedEntities =
+        this.getEntities().stream().map(entity -> entity.getName()).collect(Collectors.toList());
+    Collections.sort(sortedEntities);
+
+    List<FeatureV2> sortedFeatures = new ArrayList(this.getFeatures());
+    List<FeatureSpecV2> sortedFeatureSpecs =
+        sortedFeatures.stream().map(featureV2 -> featureV2.toProto()).collect(Collectors.toList());
+    sortedFeatures.sort(Comparator.comparing(FeatureV2::getName));
+
+    DataSourceProto.DataSource streamSource = DataSourceProto.DataSource.getDefaultInstance();
+    if (getStreamSource() != null) {
+      streamSource = getStreamSource().toProto();
+    }
+
+    FeatureTableSpec featureTableSpec =
+        FeatureTableSpec.newBuilder()
+            .addAllEntities(sortedEntities)
+            .addAllFeatures(sortedFeatureSpecs)
+            .setBatchSource(getBatchSource().toProto())
+            .setStreamSource(streamSource)
+            .setMaxAge(Duration.newBuilder().setSeconds(getMaxAgeSecs()).build())
+            .build();
+    return Hashing.murmur3_32().hashBytes(featureTableSpec.toByteArray()).toString();
+  }
+
   @Override
   public int hashCode() {
     return Objects.hash(

diff --git a/core/src/main/resources/db/migration/V3.0__Feature_Table_Deletion.sql b/core/src/main/resources/db/migration/V3.0__Feature_Table_Deletion.sql
@@ -1,3 +1 @@
-ALTER TABLE feature_tables ADD COLUMN is_deleted boolean NOT NULL;
-
-ALTER TABLE feature_tables ADD COLUMN metadata_hash bigint NOT NULL;
+ALTER TABLE feature_tables ADD COLUMN is_deleted boolean NOT NULL;
diff --git a/protos/feast/core/FeatureTable.proto b/protos/feast/core/FeatureTable.proto
@@ -79,5 +79,5 @@ message FeatureTableMeta {
 
     // Hash entities, features, batch_source and stream_source to inform JobService if
     // jobs should be restarted should hash change
-    bytes hash = 4;
+    string hash = 4;
 }