Merge pull request #158 from vitrivr/dev

v3.0.2

build.gradle

@@ -5,7 +5,7 @@ plugins {
 
 allprojects {
     group = 'org.vitrivr'
-    version = '3.0.1'
+    version = '3.0.2'
 }
 
 project.ext.protobufVersion = "3.9.0"

cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java

@@ -183,6 +183,12 @@ default List<PrimitiveTypeProvider> getUniqueValues(String column) {
     return Lists.newArrayList(uniques);
   }
 
+  default Map<String, Integer> countDistinctValues(String column) {
+    Map<String, Integer> count = new HashMap<>();
+    this.getAll(column).forEach(el -> count.compute(el.getString(), (k, v) -> v == null ? 1 : v++));
+    return count;
+  }
+
   /**
    * SELECT column from the table. Be careful with large entities
    */

cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java

@@ -157,31 +157,31 @@ public boolean createSegmentEntity() {
      * called "id", which is of type "string" and has an index. Also, for each of the provided feature attribute a field of the type "vector"
      * will be created.
      *
-     * @param featurename ame of the new entity.
+     * @param featureEntityName ame of the new entity.
      * @param unique Whether or not the provided feature should be unique per id.
      * @param featureNames List of the feature names.
      * @return True on success, false otherwise.
      */
     @Override
-    public boolean createFeatureEntity(String featurename, boolean unique, int length,
+    public boolean createFeatureEntity(String featureEntityName, boolean unique, int length,
         String... featureNames) {
         final AttributeDefinition[] attributes = Arrays.stream(featureNames)
             .map(s -> new AttributeDefinition(s, AttributeDefinition.AttributeType.VECTOR))
             .toArray(AttributeDefinition[]::new);
-        return this.createFeatureEntity(featurename, unique, attributes);
+        return this.createFeatureEntity(featureEntityName, unique, attributes);
     }
 
     /**
      * Creates and initializes a new feature entity with the provided name and the provided attributes. The new entity will have a field
      * called "id", which is of type "string" and has an index.
      *
-     * @param featurename Name of the new entity.
+     * @param featureEntityName Name of the new entity.
      * @param unique Whether or not the provided feature should be unique per id.
      * @param attributes List of {@link AttributeDefinition} objects specifying the new entities attributes.
      * @return True on success, false otherwise.
      */
     @Override
-    public boolean createFeatureEntity(String featurename, boolean unique, AttributeDefinition... attributes) {
+    public boolean createFeatureEntity(String featureEntityName, boolean unique, AttributeDefinition... attributes) {
         final AttributeDefinition[] extended = new AttributeDefinition[attributes.length + 1];
         final HashMap<String,String> hints = new HashMap<>(1);
         hints.put("indexed", "true");
@@ -195,7 +195,7 @@ public boolean createFeatureEntity(String featurename, boolean unique, Attribute
         hints.put("handler", handler);
         extended[0] = new AttributeDefinition("id", AttributeDefinition.AttributeType.STRING, hints);
         System.arraycopy(attributes, 0, extended, 1, attributes.length);
-        return this.createEntity(featurename, extended);
+        return this.createEntity(featureEntityName, extended);
     }
 
     /**

cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java

@@ -5,9 +5,7 @@
 import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.TEXT;
 import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.VECTOR;
 
-import org.vitrivr.cottontail.grpc.CottontailGrpc;
 import org.vitrivr.cottontail.grpc.CottontailGrpc.*;
-import org.vitrivr.cottontail.grpc.CottontailGrpc.Index.IndexType;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -131,11 +129,20 @@ public boolean createIndex(String entityName, String attribute, IndexType type)
         .setName("index-" + type.name().toLowerCase() + "-" + entity.getSchema().getName() + "_" + entity.getName() + "_" + attribute)
         .setType(type).build();
     /* Cottontail ignores index params as of july 19 */
-    CreateIndexMessage idxMessage = CreateIndexMessage.newBuilder().setIndex(index).addColumns(attribute).build();
+    IndexDefinition idxMessage = IndexDefinition.newBuilder().setIndex(index).addColumns(attribute).build();
     cottontail.createIndexBlocking(idxMessage);
     return true;
   }
 
+  public boolean dropIndex(String entityName, String attribute, IndexType type){
+    Entity entity = CottontailMessageBuilder.entity(entityName);
+    Index index = Index.newBuilder().setEntity(entity)
+        .setName("index-" + type.name().toLowerCase() + "-" + entity.getSchema().getName() + "_" + entity.getName() + "_" + attribute)
+        .setType(type).build();
+    cottontail.dropIndexBlocking(index);
+    return true;
+  }
+
   @Override
   public boolean createSegmentEntity() {
     ArrayList<ColumnDefinition> columns = new ArrayList<>(4);
@@ -162,23 +169,23 @@ public boolean createSegmentEntity() {
   }
 
   @Override
-  public boolean createFeatureEntity(String featurename, boolean unique, int length,
+  public boolean createFeatureEntity(String featureEntityName, boolean unique, int length,
       String... featureNames) {
     final AttributeDefinition[] attributes = Arrays.stream(featureNames)
         .map(s -> new AttributeDefinition(s, VECTOR, length))
         .toArray(AttributeDefinition[]::new);
-    return this.createFeatureEntity(featurename, unique, attributes);
+    return this.createFeatureEntity(featureEntityName, unique, attributes);
   }
 
   @Override
-  public boolean createFeatureEntity(String featurename, boolean unique,
+  public boolean createFeatureEntity(String featureEntityName, boolean unique,
       AttributeDefinition... attributes) {
     final AttributeDefinition[] extended = new AttributeDefinition[attributes.length + 1];
     final HashMap<String, String> hints = new HashMap<>(1);
 
     extended[0] = new AttributeDefinition("id", AttributeDefinition.AttributeType.STRING, hints);
     System.arraycopy(attributes, 0, extended, 1, attributes.length);
-    return this.createEntity(featurename, extended);
+    return this.createEntity(featureEntityName, extended);
   }
 
   @Override

cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java

@@ -5,12 +5,11 @@
 import static org.vitrivr.cineast.core.db.cottontaildb.CottontailMessageBuilder.toDatas;
 import static org.vitrivr.cineast.core.db.cottontaildb.CottontailMessageBuilder.whereInList;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
+
 import org.apache.commons.lang3.time.StopWatch;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.commons.lang3.tuple.Triple;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -181,6 +180,18 @@ public List<PrimitiveTypeProvider> getUniqueValues(String column) {
     return toSingleCol(results, column);
   }
 
+  public Map<String, Integer> countDistinctValues(String column) {
+    QueryMessage queryMessage = queryMessage(
+        query(entity, CottontailMessageBuilder.projection(Operation.SELECT, column), null, null,
+            null), null);
+    Map<String, Integer> count = new HashMap<>();
+    List<QueryResponseMessage> list = this.cottontail.query(queryMessage);
+    list.forEach(row -> row.getResultsList().forEach(tuple -> {
+      count.merge(tuple.getDataMap().get(column).getStringData(), 1, (old, one) -> old + 1);
+    }));
+    return count;
+  }
+
   @Override
   public List<Map<String, PrimitiveTypeProvider>> getAll() {
     QueryMessage queryMessage = queryMessage(query(entity, SELECT_ALL_PROJECTION, null, null, null), null);

cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java

@@ -32,7 +32,7 @@
 public class CottontailWrapper implements AutoCloseable {
 
   private static final Logger LOGGER = LogManager.getLogger();
-  private static final InsertStatus INTERRUPTED_INSERT = InsertStatus.newBuilder().setSuccess(false).build();
+  private static final CottontailGrpc.Status INTERRUPTED_INSERT = CottontailGrpc.Status.newBuilder().setSuccess(false).build();
 
   private final ManagedChannel channel;
   private final CottonDDLFutureStub definitionFutureStub;
@@ -60,30 +60,30 @@ public CottontailWrapper(DatabaseConfig config, boolean closeWrapper) {
     LOGGER.info("Connected to Cottontail in {} ms at {}:{}", watch.getTime(TimeUnit.MILLISECONDS), config.getHost(), config.getPort());
   }
 
-  public synchronized ListenableFuture<SuccessStatus> createEntity(EntityDefinition createMessage) {
+  public synchronized ListenableFuture<CottontailGrpc.Status> createEntity(EntityDefinition createMessage) {
     final CottonDDLFutureStub stub = CottonDDLGrpc.newFutureStub(this.channel);
     return stub.createEntity(createMessage);
   }
 
-  public synchronized ListenableFuture<CottontailGrpc.EntityDefinition> entityDetails(Entity entity){
+  public synchronized ListenableFuture<CottontailGrpc.EntityDefinition> entityDetails(Entity entity) {
     return CottonDDLGrpc.newFutureStub(this.channel).entityDetails(entity);
   }
 
-  public synchronized CottontailGrpc.EntityDefinition entityDetailsBlocking(Entity entity){
+  public synchronized CottontailGrpc.EntityDefinition entityDetailsBlocking(Entity entity) {
     final CottonDDLBlockingStub stub = CottonDDLGrpc.newBlockingStub(this.channel);
-    try{
+    try {
       return stub.entityDetails(entity);
-    }catch(StatusRuntimeException e){
-      if(e.getStatus().getCode() == Status.NOT_FOUND.getCode()){
+    } catch (StatusRuntimeException e) {
+      if (e.getStatus().getCode() == Status.NOT_FOUND.getCode()) {
         LOGGER.warn("Entity {} was not found", entity);
         return null;
-      }else{
+      } else {
         throw LOGGER.throwing(e);
       }
     }
   }
 
-  public static Entity entityByName(String entityName){
+  public static Entity entityByName(String entityName) {
     return CottontailMessageBuilder.entity(CottontailMessageBuilder.CINEAST_SCHEMA, entityName);
   }
 
@@ -102,14 +102,30 @@ public synchronized boolean createEntityBlocking(EntityDefinition createMessage)
     return false;
   }
 
-  public synchronized boolean createIndexBlocking(CreateIndexMessage createMessage) {
+  public synchronized boolean createIndexBlocking(IndexDefinition createMessage) {
     final CottonDDLBlockingStub stub = CottonDDLGrpc.newBlockingStub(this.channel);
     try {
       stub.createIndex(createMessage);
       return true;
     } catch (StatusRuntimeException e) {
       if (e.getStatus().getCode() == Status.ALREADY_EXISTS.getCode()) {
         LOGGER.warn("Index on {}.{} was not created because it already exists", createMessage.getIndex().getEntity().getName(), createMessage.getColumnsList().toString());
+        return false;
+      }
+      e.printStackTrace();
+    }
+    return false;
+  }
+
+  public synchronized boolean dropIndexBlocking(Index index) {
+    final CottonDDLBlockingStub stub = CottonDDLGrpc.newBlockingStub(this.channel);
+    try {
+      stub.dropIndex(index);
+      return true;
+    } catch (StatusRuntimeException e) {
+      if (e.getStatus() == Status.NOT_FOUND) {
+        LOGGER.warn("Index {} was not dropped because it does not exist", index.getName());
+        return false;
       }
       e.printStackTrace();
     }
@@ -119,7 +135,7 @@ public synchronized boolean createIndexBlocking(CreateIndexMessage createMessage
   public synchronized boolean optimizeEntityBlocking(Entity entity) {
     final CottonDDLBlockingStub stub = CottonDDLGrpc.newBlockingStub(this.channel);
     try {
-      stub.optimizeEntity(entity);
+      stub.optimize(entity);
       return true;
     } catch (StatusRuntimeException e) {
       e.printStackTrace();
@@ -142,13 +158,13 @@ public synchronized boolean dropEntityBlocking(Entity entity) {
     return false;
   }
 
-  public synchronized ListenableFuture<SuccessStatus> createSchema(String schama) {
+  public synchronized ListenableFuture<CottontailGrpc.Status> createSchema(String schama) {
     final CottonDDLFutureStub stub = CottonDDLGrpc.newFutureStub(this.channel);
     return stub.createSchema(CottontailMessageBuilder.schema(schama));
   }
 
   public synchronized boolean createSchemaBlocking(String schema) {
-    ListenableFuture<SuccessStatus> future = this.createSchema(schema);
+    ListenableFuture<CottontailGrpc.Status> future = this.createSchema(schema);
     try {
       future.get();
       return true;
@@ -180,10 +196,10 @@ public synchronized void ensureSchemaBlocking(String schema) {
   public boolean insert(List<InsertMessage> messages) {
 
     final boolean[] status = {false, false}; /* {done, error}. */
-    final StreamObserver<InsertStatus> observer = new StreamObserver<InsertStatus>() {
+    final StreamObserver<CottontailGrpc.Status> observer = new StreamObserver<CottontailGrpc.Status>() {
 
       @Override
-      public void onNext(InsertStatus value) {
+      public void onNext(CottontailGrpc.Status value) {
         LOGGER.trace("Tuple received: {}", value.getTimestamp());
       }
 
@@ -264,7 +280,7 @@ public List<QueryResponseMessage> batchedQuery(BatchedQueryMessage query) {
   public boolean ping() {
     final CottonDQLBlockingStub stub = CottonDQLGrpc.newBlockingStub(this.channel).withDeadlineAfter(MAX_CALL_TIMEOUT, TimeUnit.MILLISECONDS);
     try {
-      final SuccessStatus status = stub.ping(Empty.getDefaultInstance());
+      final CottontailGrpc.Status status = stub.ping(Empty.getDefaultInstance());
       return true;
     } catch (StatusRuntimeException e) {
       if (e.getStatus() == Status.DEADLINE_EXCEEDED) {

cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java

@@ -2,6 +2,7 @@
 
 import org.vitrivr.cottontail.grpc.CottontailGrpc.Data;
 import org.vitrivr.cottontail.grpc.CottontailGrpc.Entity;
+import org.vitrivr.cottontail.grpc.CottontailGrpc.From;
 import org.vitrivr.cottontail.grpc.CottontailGrpc.InsertMessage;
 import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection;
 import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection.Operation;
@@ -63,7 +64,7 @@ public boolean exists(String key, String value) {
     @Override
     public boolean persist(List<PersistentTuple> tuples) {
         final List<InsertMessage> messages = tuples.stream()
-            .map(t -> InsertMessage.newBuilder().setEntity(this.entity).setTuple(this.getPersistentRepresentation(t)).build())
+            .map(t -> InsertMessage.newBuilder().setFrom(From.newBuilder().setEntity(this.entity)).setTuple(this.getPersistentRepresentation(t)).build())
             .collect(Collectors.toList());
         return this.cottontail.insert(messages);
     }

cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryEntityCreator.java

@@ -63,16 +63,16 @@ public boolean createSegmentEntity() {
   }
 
   @Override
-  public boolean createFeatureEntity(String featurename, boolean unique, int length, String... featureNames) {
+  public boolean createFeatureEntity(String featureEntityName, boolean unique, int length, String... featureNames) {
     final String[] columns = new String[featureNames.length + 1];
     columns[0] = "id";
     System.arraycopy(featureNames, 0, columns, 1, columns.length - 1);
-    return this.store.createEntity(featurename, columns).isPresent();
+    return this.store.createEntity(featureEntityName, columns).isPresent();
   }
 
   @Override
-  public boolean createFeatureEntity(String featurename, boolean unique, AttributeDefinition... attributes) {
-    return createIdEntity(featurename, attributes);
+  public boolean createFeatureEntity(String featureEntityName, boolean unique, AttributeDefinition... attributes) {
+    return createIdEntity(featureEntityName, attributes);
   }
 
   @Override

cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityCreator.java

@@ -117,16 +117,23 @@ default boolean dropTagEntity() {
     /**
      * Creates and initializes an entity for a feature module with default parameters
      *
-     * @param featurename the name of the feature module
+     * @param featureEntityName the name of the feature module
      * @param unique      true if the feature module produces at most one vector per segment
      */
-    default boolean createFeatureEntity(String featurename, boolean unique, int length) {
-        return createFeatureEntity(featurename, unique, length, "feature");
+    default boolean createFeatureEntity(String featureEntityName, boolean unique, int length) {
+        return createFeatureEntity(featureEntityName, unique, length, "feature");
     }
 
-    boolean createFeatureEntity(String featurename, boolean unique, int length, String... featureNames);
+    boolean createFeatureEntity(String featureEntityName, boolean unique, int length, String... featureNames);
 
-    boolean createFeatureEntity(String featurename, boolean unique, AttributeDefinition... attributes);
+    /**
+     * Creates and initializes an entity for a feature module with default parameters
+     *
+     * @param featureEntityName the name of the feature module
+     * @param unique      true if the feature module produces at most one vector per segment
+     * @param attributes description of the columns besides the id column
+     */
+    boolean createFeatureEntity(String featureEntityName, boolean unique, AttributeDefinition... attributes);
 
     /**
      * Creates and initializes an entity with the provided name and the provided attributes. The new entity will have an additional

cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java

@@ -39,14 +39,14 @@ public boolean dropMetadataEntity() {
   }
 
   @Override
-  public boolean createFeatureEntity(String featurename, boolean unique, int length,
+  public boolean createFeatureEntity(String featureEntityName, boolean unique, int length,
       String... featureNames) {
     return false;
   }
 
 
   @Override
-  public boolean createFeatureEntity(String featurename, boolean unique,
+  public boolean createFeatureEntity(String featureEntityName, boolean unique,
       AttributeDefinition... attributes) {
     return false;
   }

cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java

@@ -65,7 +65,7 @@ private void populateExtremaMap(){
         if (comparator.compare(t, min) < 0){
           min = t;
         }
-        if(comparator.compare(max, t) > 0){
+        if(comparator.compare(t, max) > 0){
           max = t;
         }
       }

cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java

@@ -58,6 +58,7 @@ public void initalizePersistentLayer(Supplier<EntityCreator> supply) {
         new AttributeDefinition("score", AttributeType.FLOAT));
 
     supply.get().createHashNonUniqueIndex(SEGMENT_TAGS_TABLE_NAME, "tagid");
+    supply.get().createHashNonUniqueIndex(SEGMENT_TAGS_TABLE_NAME, "id");
   }
 
   @Override