diff --git a/build.gradle b/build.gradle index 489dbcd99..67f3da287 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ plugins { allprojects { group = 'org.vitrivr' - version = '3.0.1' + version = '3.0.2' } project.ext.protobufVersion = "3.9.0" diff --git a/cineast-core/cottontaildb-proto b/cineast-core/cottontaildb-proto index 6f3f20f27..b29acbf39 160000 --- a/cineast-core/cottontaildb-proto +++ b/cineast-core/cottontaildb-proto @@ -1 +1 @@ -Subproject commit 6f3f20f27825c42edf4f7ac6b71013558fa3ed64 +Subproject commit b29acbf390f83fe2a936ee48988241350a1e2fbe diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java index 10d030d03..ac4775232 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java @@ -183,6 +183,12 @@ default List getUniqueValues(String column) { return Lists.newArrayList(uniques); } + default Map countDistinctValues(String column) { + Map count = new HashMap<>(); + this.getAll(column).forEach(el -> count.compute(el.getString(), (k, v) -> v == null ? 1 : v++)); + return count; + } + /** * SELECT column from the table. Be careful with large entities */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java index 6cb43987a..d9b41e0cf 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java @@ -157,31 +157,31 @@ public boolean createSegmentEntity() { * called "id", which is of type "string" and has an index. Also, for each of the provided feature attribute a field of the type "vector" * will be created. * - * @param featurename ame of the new entity. + * @param featureEntityName ame of the new entity. * @param unique Whether or not the provided feature should be unique per id. * @param featureNames List of the feature names. * @return True on success, false otherwise. */ @Override - public boolean createFeatureEntity(String featurename, boolean unique, int length, + public boolean createFeatureEntity(String featureEntityName, boolean unique, int length, String... featureNames) { final AttributeDefinition[] attributes = Arrays.stream(featureNames) .map(s -> new AttributeDefinition(s, AttributeDefinition.AttributeType.VECTOR)) .toArray(AttributeDefinition[]::new); - return this.createFeatureEntity(featurename, unique, attributes); + return this.createFeatureEntity(featureEntityName, unique, attributes); } /** * Creates and initializes a new feature entity with the provided name and the provided attributes. The new entity will have a field * called "id", which is of type "string" and has an index. * - * @param featurename Name of the new entity. + * @param featureEntityName Name of the new entity. * @param unique Whether or not the provided feature should be unique per id. * @param attributes List of {@link AttributeDefinition} objects specifying the new entities attributes. * @return True on success, false otherwise. */ @Override - public boolean createFeatureEntity(String featurename, boolean unique, AttributeDefinition... attributes) { + public boolean createFeatureEntity(String featureEntityName, boolean unique, AttributeDefinition... attributes) { final AttributeDefinition[] extended = new AttributeDefinition[attributes.length + 1]; final HashMap hints = new HashMap<>(1); hints.put("indexed", "true"); @@ -195,7 +195,7 @@ public boolean createFeatureEntity(String featurename, boolean unique, Attribute hints.put("handler", handler); extended[0] = new AttributeDefinition("id", AttributeDefinition.AttributeType.STRING, hints); System.arraycopy(attributes, 0, extended, 1, attributes.length); - return this.createEntity(featurename, extended); + return this.createEntity(featureEntityName, extended); } /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java index 1bad4cbb1..bd77bd818 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java @@ -5,9 +5,7 @@ import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.TEXT; import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.VECTOR; -import org.vitrivr.cottontail.grpc.CottontailGrpc; import org.vitrivr.cottontail.grpc.CottontailGrpc.*; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Index.IndexType; import java.util.ArrayList; import java.util.Arrays; @@ -131,11 +129,20 @@ public boolean createIndex(String entityName, String attribute, IndexType type) .setName("index-" + type.name().toLowerCase() + "-" + entity.getSchema().getName() + "_" + entity.getName() + "_" + attribute) .setType(type).build(); /* Cottontail ignores index params as of july 19 */ - CreateIndexMessage idxMessage = CreateIndexMessage.newBuilder().setIndex(index).addColumns(attribute).build(); + IndexDefinition idxMessage = IndexDefinition.newBuilder().setIndex(index).addColumns(attribute).build(); cottontail.createIndexBlocking(idxMessage); return true; } + public boolean dropIndex(String entityName, String attribute, IndexType type){ + Entity entity = CottontailMessageBuilder.entity(entityName); + Index index = Index.newBuilder().setEntity(entity) + .setName("index-" + type.name().toLowerCase() + "-" + entity.getSchema().getName() + "_" + entity.getName() + "_" + attribute) + .setType(type).build(); + cottontail.dropIndexBlocking(index); + return true; + } + @Override public boolean createSegmentEntity() { ArrayList columns = new ArrayList<>(4); @@ -162,23 +169,23 @@ public boolean createSegmentEntity() { } @Override - public boolean createFeatureEntity(String featurename, boolean unique, int length, + public boolean createFeatureEntity(String featureEntityName, boolean unique, int length, String... featureNames) { final AttributeDefinition[] attributes = Arrays.stream(featureNames) .map(s -> new AttributeDefinition(s, VECTOR, length)) .toArray(AttributeDefinition[]::new); - return this.createFeatureEntity(featurename, unique, attributes); + return this.createFeatureEntity(featureEntityName, unique, attributes); } @Override - public boolean createFeatureEntity(String featurename, boolean unique, + public boolean createFeatureEntity(String featureEntityName, boolean unique, AttributeDefinition... attributes) { final AttributeDefinition[] extended = new AttributeDefinition[attributes.length + 1]; final HashMap hints = new HashMap<>(1); extended[0] = new AttributeDefinition("id", AttributeDefinition.AttributeType.STRING, hints); System.arraycopy(attributes, 0, extended, 1, attributes.length); - return this.createEntity(featurename, extended); + return this.createEntity(featureEntityName, extended); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index cd5d24345..9b87acbd7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -5,12 +5,11 @@ import static org.vitrivr.cineast.core.db.cottontaildb.CottontailMessageBuilder.toDatas; import static org.vitrivr.cineast.core.db.cottontaildb.CottontailMessageBuilder.whereInList; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.TimeUnit; + import org.apache.commons.lang3.time.StopWatch; +import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -181,6 +180,18 @@ public List getUniqueValues(String column) { return toSingleCol(results, column); } + public Map countDistinctValues(String column) { + QueryMessage queryMessage = queryMessage( + query(entity, CottontailMessageBuilder.projection(Operation.SELECT, column), null, null, + null), null); + Map count = new HashMap<>(); + List list = this.cottontail.query(queryMessage); + list.forEach(row -> row.getResultsList().forEach(tuple -> { + count.merge(tuple.getDataMap().get(column).getStringData(), 1, (old, one) -> old + 1); + })); + return count; + } + @Override public List> getAll() { QueryMessage queryMessage = queryMessage(query(entity, SELECT_ALL_PROJECTION, null, null, null), null); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java index 033e45850..87020fce8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java @@ -32,7 +32,7 @@ public class CottontailWrapper implements AutoCloseable { private static final Logger LOGGER = LogManager.getLogger(); - private static final InsertStatus INTERRUPTED_INSERT = InsertStatus.newBuilder().setSuccess(false).build(); + private static final CottontailGrpc.Status INTERRUPTED_INSERT = CottontailGrpc.Status.newBuilder().setSuccess(false).build(); private final ManagedChannel channel; private final CottonDDLFutureStub definitionFutureStub; @@ -60,30 +60,30 @@ public CottontailWrapper(DatabaseConfig config, boolean closeWrapper) { LOGGER.info("Connected to Cottontail in {} ms at {}:{}", watch.getTime(TimeUnit.MILLISECONDS), config.getHost(), config.getPort()); } - public synchronized ListenableFuture createEntity(EntityDefinition createMessage) { + public synchronized ListenableFuture createEntity(EntityDefinition createMessage) { final CottonDDLFutureStub stub = CottonDDLGrpc.newFutureStub(this.channel); return stub.createEntity(createMessage); } - public synchronized ListenableFuture entityDetails(Entity entity){ + public synchronized ListenableFuture entityDetails(Entity entity) { return CottonDDLGrpc.newFutureStub(this.channel).entityDetails(entity); } - public synchronized CottontailGrpc.EntityDefinition entityDetailsBlocking(Entity entity){ + public synchronized CottontailGrpc.EntityDefinition entityDetailsBlocking(Entity entity) { final CottonDDLBlockingStub stub = CottonDDLGrpc.newBlockingStub(this.channel); - try{ + try { return stub.entityDetails(entity); - }catch(StatusRuntimeException e){ - if(e.getStatus().getCode() == Status.NOT_FOUND.getCode()){ + } catch (StatusRuntimeException e) { + if (e.getStatus().getCode() == Status.NOT_FOUND.getCode()) { LOGGER.warn("Entity {} was not found", entity); return null; - }else{ + } else { throw LOGGER.throwing(e); } } } - public static Entity entityByName(String entityName){ + public static Entity entityByName(String entityName) { return CottontailMessageBuilder.entity(CottontailMessageBuilder.CINEAST_SCHEMA, entityName); } @@ -102,7 +102,7 @@ public synchronized boolean createEntityBlocking(EntityDefinition createMessage) return false; } - public synchronized boolean createIndexBlocking(CreateIndexMessage createMessage) { + public synchronized boolean createIndexBlocking(IndexDefinition createMessage) { final CottonDDLBlockingStub stub = CottonDDLGrpc.newBlockingStub(this.channel); try { stub.createIndex(createMessage); @@ -110,6 +110,22 @@ public synchronized boolean createIndexBlocking(CreateIndexMessage createMessage } catch (StatusRuntimeException e) { if (e.getStatus().getCode() == Status.ALREADY_EXISTS.getCode()) { LOGGER.warn("Index on {}.{} was not created because it already exists", createMessage.getIndex().getEntity().getName(), createMessage.getColumnsList().toString()); + return false; + } + e.printStackTrace(); + } + return false; + } + + public synchronized boolean dropIndexBlocking(Index index) { + final CottonDDLBlockingStub stub = CottonDDLGrpc.newBlockingStub(this.channel); + try { + stub.dropIndex(index); + return true; + } catch (StatusRuntimeException e) { + if (e.getStatus() == Status.NOT_FOUND) { + LOGGER.warn("Index {} was not dropped because it does not exist", index.getName()); + return false; } e.printStackTrace(); } @@ -119,7 +135,7 @@ public synchronized boolean createIndexBlocking(CreateIndexMessage createMessage public synchronized boolean optimizeEntityBlocking(Entity entity) { final CottonDDLBlockingStub stub = CottonDDLGrpc.newBlockingStub(this.channel); try { - stub.optimizeEntity(entity); + stub.optimize(entity); return true; } catch (StatusRuntimeException e) { e.printStackTrace(); @@ -142,13 +158,13 @@ public synchronized boolean dropEntityBlocking(Entity entity) { return false; } - public synchronized ListenableFuture createSchema(String schama) { + public synchronized ListenableFuture createSchema(String schama) { final CottonDDLFutureStub stub = CottonDDLGrpc.newFutureStub(this.channel); return stub.createSchema(CottontailMessageBuilder.schema(schama)); } public synchronized boolean createSchemaBlocking(String schema) { - ListenableFuture future = this.createSchema(schema); + ListenableFuture future = this.createSchema(schema); try { future.get(); return true; @@ -180,10 +196,10 @@ public synchronized void ensureSchemaBlocking(String schema) { public boolean insert(List messages) { final boolean[] status = {false, false}; /* {done, error}. */ - final StreamObserver observer = new StreamObserver() { + final StreamObserver observer = new StreamObserver() { @Override - public void onNext(InsertStatus value) { + public void onNext(CottontailGrpc.Status value) { LOGGER.trace("Tuple received: {}", value.getTimestamp()); } @@ -264,7 +280,7 @@ public List batchedQuery(BatchedQueryMessage query) { public boolean ping() { final CottonDQLBlockingStub stub = CottonDQLGrpc.newBlockingStub(this.channel).withDeadlineAfter(MAX_CALL_TIMEOUT, TimeUnit.MILLISECONDS); try { - final SuccessStatus status = stub.ping(Empty.getDefaultInstance()); + final CottontailGrpc.Status status = stub.ping(Empty.getDefaultInstance()); return true; } catch (StatusRuntimeException e) { if (e.getStatus() == Status.DEADLINE_EXCEEDED) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java index b65c9d562..9dac446e0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java @@ -2,6 +2,7 @@ import org.vitrivr.cottontail.grpc.CottontailGrpc.Data; import org.vitrivr.cottontail.grpc.CottontailGrpc.Entity; +import org.vitrivr.cottontail.grpc.CottontailGrpc.From; import org.vitrivr.cottontail.grpc.CottontailGrpc.InsertMessage; import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection; import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection.Operation; @@ -63,7 +64,7 @@ public boolean exists(String key, String value) { @Override public boolean persist(List tuples) { final List messages = tuples.stream() - .map(t -> InsertMessage.newBuilder().setEntity(this.entity).setTuple(this.getPersistentRepresentation(t)).build()) + .map(t -> InsertMessage.newBuilder().setFrom(From.newBuilder().setEntity(this.entity)).setTuple(this.getPersistentRepresentation(t)).build()) .collect(Collectors.toList()); return this.cottontail.insert(messages); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryEntityCreator.java index bdc1ec13d..f392f3af0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryEntityCreator.java @@ -63,16 +63,16 @@ public boolean createSegmentEntity() { } @Override - public boolean createFeatureEntity(String featurename, boolean unique, int length, String... featureNames) { + public boolean createFeatureEntity(String featureEntityName, boolean unique, int length, String... featureNames) { final String[] columns = new String[featureNames.length + 1]; columns[0] = "id"; System.arraycopy(featureNames, 0, columns, 1, columns.length - 1); - return this.store.createEntity(featurename, columns).isPresent(); + return this.store.createEntity(featureEntityName, columns).isPresent(); } @Override - public boolean createFeatureEntity(String featurename, boolean unique, AttributeDefinition... attributes) { - return createIdEntity(featurename, attributes); + public boolean createFeatureEntity(String featureEntityName, boolean unique, AttributeDefinition... attributes) { + return createIdEntity(featureEntityName, attributes); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityCreator.java index 6fc76552e..5f7d97b0e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityCreator.java @@ -117,16 +117,23 @@ default boolean dropTagEntity() { /** * Creates and initializes an entity for a feature module with default parameters * - * @param featurename the name of the feature module + * @param featureEntityName the name of the feature module * @param unique true if the feature module produces at most one vector per segment */ - default boolean createFeatureEntity(String featurename, boolean unique, int length) { - return createFeatureEntity(featurename, unique, length, "feature"); + default boolean createFeatureEntity(String featureEntityName, boolean unique, int length) { + return createFeatureEntity(featureEntityName, unique, length, "feature"); } - boolean createFeatureEntity(String featurename, boolean unique, int length, String... featureNames); + boolean createFeatureEntity(String featureEntityName, boolean unique, int length, String... featureNames); - boolean createFeatureEntity(String featurename, boolean unique, AttributeDefinition... attributes); + /** + * Creates and initializes an entity for a feature module with default parameters + * + * @param featureEntityName the name of the feature module + * @param unique true if the feature module produces at most one vector per segment + * @param attributes description of the columns besides the id column + */ + boolean createFeatureEntity(String featureEntityName, boolean unique, AttributeDefinition... attributes); /** * Creates and initializes an entity with the provided name and the provided attributes. The new entity will have an additional diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java index e05942823..e9c38e0b6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java @@ -39,14 +39,14 @@ public boolean dropMetadataEntity() { } @Override - public boolean createFeatureEntity(String featurename, boolean unique, int length, + public boolean createFeatureEntity(String featureEntityName, boolean unique, int length, String... featureNames) { return false; } @Override - public boolean createFeatureEntity(String featurename, boolean unique, + public boolean createFeatureEntity(String featureEntityName, boolean unique, AttributeDefinition... attributes) { return false; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java index d0ce62c70..8040c0bfb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java @@ -65,7 +65,7 @@ private void populateExtremaMap(){ if (comparator.compare(t, min) < 0){ min = t; } - if(comparator.compare(max, t) > 0){ + if(comparator.compare(t, max) > 0){ max = t; } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java index b33200d0f..ce1055263 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java @@ -58,6 +58,7 @@ public void initalizePersistentLayer(Supplier supply) { new AttributeDefinition("score", AttributeType.FLOAT)); supply.get().createHashNonUniqueIndex(SEGMENT_TAGS_TABLE_NAME, "tagid"); + supply.get().createHashNonUniqueIndex(SEGMENT_TAGS_TABLE_NAME, "id"); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/BooleanRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/BooleanRetriever.java index 40593aaeb..fe79d8e22 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/BooleanRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/BooleanRetriever.java @@ -84,8 +84,17 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc protected List getMatching(List expressions, ReadableQueryConfig qc){ - List> rows = selector.getRowsAND(expressions.stream().map(be -> Triple.of(be.getAttribute().contains(this.entity) ? be.getAttribute().substring(this.entity.length()+1) : be.getAttribute(), be.getOperator(), be.getValues())).collect(Collectors.toList()), "id", Collections.singletonList("id"), qc); - + List> rows = selector.getRowsAND( + expressions.stream().map(be -> Triple.of( + // strip entity if it was given via config + be.getAttribute().contains(this.entity) ? be.getAttribute().substring(this.entity.length()+1) : be.getAttribute(), + be.getOperator(), + be.getValues() + )).collect(Collectors.toList()), + "id", // for compound ops, we want to join via id. Cottontail (the official storage layer) does not use this identifier + Collections.singletonList("id"), // we're only interested in the ids + qc); + // we're returning a boolean score element since the score is always 1 if a query matches here return rows.stream().map(row -> new BooleanSegmentScoreElement(row.get("id").getString())).collect(Collectors.toList()); } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/Main.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/Main.java index 1085eed58..e6aee3618 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/Main.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/Main.java @@ -13,20 +13,17 @@ public class Main { * @param args Program arguments. */ public static void main(String[] args) { - /* (Force) load application config. */ - if (args.length < 2) { - System.err.println("Starting Cineast requires at least two arguments: the path to the configuration file and a command (cineast . Cineast will shutdown..."); - System.exit(1); - } - /* (Force) load application config. */ if (Config.loadConfig(args[0]) == null) { System.err.println("Failed to load Cineast configuration from '" + args[0] + "'. Cineast will shutdown..."); System.exit(1); } + if(args.length==1){ + CLI.start(CineastCli.class); + } /* Either start Cineast in interactive mode OR execute command directly. */ - if (args[1].equals("interactive")) { + if(args.length==2 && args[1].equals("interactive")){ CLI.start(CineastCli.class); } else { com.github.rvesse.airline.Cli cli = new com.github.rvesse.airline.Cli<>(CineastCli.class); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CineastCli.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CineastCli.java index bab22f898..66a3dbf4d 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CineastCli.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CineastCli.java @@ -4,7 +4,7 @@ import com.github.rvesse.airline.help.Help; import org.vitrivr.cineast.standalone.cli.db.DropTableCommand; -@Cli(name = "cineast-api", description = "The CLI provided by the Cineast API.", commands = {DropTableCommand.class, TagRetrievalCommand.class, OptimizeEntitiesCommand.class, CodebookCommand.class, DatabaseSetupCommand.class, ExtractionCommand.class, ImportCommand.class, ThreeDeeTestCommand.class, RetrieveCommand.class, Help.class, SingleObjRetrievalCommand.class, TextRetrievalCommand.class}) +@Cli(name = "cineast-api", description = "The CLI provided by the Cineast API.", commands = {DropTableCommand.class, TagRetrievalCommand.class, OptimizeEntitiesCommand.class, CodebookCommand.class, DatabaseSetupCommand.class, ExtractionCommand.class, ImportCommand.class, ThreeDeeTestCommand.class, RetrieveCommand.class, Help.class, SingleObjRetrievalCommand.class, TextRetrievalCommand.class, DistinctColumnApiCommand.class}, defaultCommand = Help.class) public class CineastCli { } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/DistinctColumnApiCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/DistinctColumnApiCommand.java new file mode 100644 index 000000000..f26d199ce --- /dev/null +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/DistinctColumnApiCommand.java @@ -0,0 +1,44 @@ +package org.vitrivr.cineast.standalone.cli; + +import com.github.rvesse.airline.annotations.Command; +import com.github.rvesse.airline.annotations.Option; +import com.github.rvesse.airline.annotations.restrictions.Required; +import java.util.Comparator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.ToIntFunction; +import org.vitrivr.cineast.core.db.DBSelector; +import org.vitrivr.cineast.standalone.config.Config; + +@Command(name = "distinct-column", description = "Retrieves all distinct elements from the database for a given table and a given column") +public class DistinctColumnApiCommand implements Runnable { + + @Option(name = {"--table"}, title = "Table", description = "Table in the underlying database") + @Required + private String table; + + @Option(name = {"--column"}, title = "Column", description = "Column of the specified table") + @Required + private String column; + + @Option(name = {"--limit"}, title = "Limit", description = "Minimum occurences to be printed") + private int limit = -1; + + @Override + public void run() { + DBSelector selector = Config.sharedConfig().getDatabase().getSelectorSupplier().get(); + selector.open(table); + long start = System.currentTimeMillis(); + Map distinct = selector.countDistinctValues(column); + long stop = System.currentTimeMillis(); + System.out.println("Retrieved distinct elements in " + (stop - start) + " ms"); + System.out.println("Printing distinct elements for " + table + "." + column); + AtomicInteger dynamicLimit = new AtomicInteger(limit); + distinct.entrySet().stream() + .filter(el -> el.getValue() >= dynamicLimit.get()) + .sorted(Comparator.comparingInt((ToIntFunction>) Entry::getValue).reversed()) + .forEach(entry -> System.out.println(entry.getKey() + ": " + entry.getValue())); + selector.close(); + } +} diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java index 0a6f83c18..c3d8282b4 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java @@ -28,4 +28,5 @@ public static void optimizeAllCottontailEntities() { }); System.out.println("Finished optimizing all entities"); } + } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/DropTableCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/DropTableCommand.java index 342894f60..6dc4c6627 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/DropTableCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/DropTableCommand.java @@ -19,6 +19,10 @@ public class DropTableCommand implements Runnable { @Override public void run() { + dropTable(tableName); + } + + public static void dropTable(String tableName) { final EntityCreator ec = Config.sharedConfig().getDatabase().getEntityCreatorSupplier().get(); if (ec != null) { System.out.println("Dropping " + tableName); diff --git a/example_job.json b/example_json_job.json similarity index 95% rename from example_job.json rename to example_json_job.json index b54272ba7..bd9f48dcb 100644 --- a/example_job.json +++ b/example_json_job.json @@ -35,7 +35,8 @@ } ], "database":{ - "writer": "PROTO", + "host": "output/", + "writer": "JSON", "selector": "NONE" } }