Skip to content

Commit

Permalink
feat: queryable index (unique)
Browse files Browse the repository at this point in the history
Indexes now support API to be queried with operations such as LT, LTE, GT, GTE

ref: Moved EngineConfig to context
  • Loading branch information
sepgh committed Oct 27, 2024
1 parent 5332d23 commit a8c0233
Show file tree
Hide file tree
Showing 32 changed files with 386 additions and 50 deletions.
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package com.github.sepgh.testudo;
package com.github.sepgh.testudo.context;


import com.github.sepgh.testudo.serialization.FieldType;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.github.sepgh.testudo.index;

import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.index.data.IndexBinaryObject;
import com.github.sepgh.testudo.index.data.IndexBinaryObjectFactory;
import com.github.sepgh.testudo.storage.db.DBObject;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.github.sepgh.testudo.index;

import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.exception.IndexExistsException;
import com.github.sepgh.testudo.exception.IndexMissingException;
import com.github.sepgh.testudo.exception.InternalOperationException;
Expand Down
14 changes: 14 additions & 0 deletions src/main/java/com/github/sepgh/testudo/index/QueryableIndex.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package com.github.sepgh.testudo.index;

import com.github.sepgh.testudo.exception.InternalOperationException;

import java.util.Iterator;
import java.util.Optional;

public interface QueryableIndex<K extends Comparable<K>, V> {
Iterator<V> getGreaterThan(K k) throws InternalOperationException;
Iterator<V> getGreaterThanEqual(K k) throws InternalOperationException;
Iterator<V> getLessThan(K k) throws InternalOperationException;
Iterator<V> getLessThanEqual(K k) throws InternalOperationException;
Optional<V> getEqual(K k) throws InternalOperationException;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package com.github.sepgh.testudo.index;

public interface UniqueQueryableIndex<K extends Comparable<K>, V>
extends UniqueTreeIndexManager<K, V>, QueryableIndex<K, V>
{}
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@
import com.github.sepgh.testudo.exception.IndexExistsException;
import com.github.sepgh.testudo.exception.IndexMissingException;
import com.github.sepgh.testudo.exception.InternalOperationException;
import com.github.sepgh.testudo.index.AbstractUniqueTreeIndexManager;
import com.github.sepgh.testudo.index.KeyValue;
import com.github.sepgh.testudo.index.Pointer;
import com.github.sepgh.testudo.index.*;
import com.github.sepgh.testudo.index.data.IndexBinaryObjectFactory;
import com.github.sepgh.testudo.index.tree.node.AbstractLeafTreeNode;
import com.github.sepgh.testudo.index.tree.node.AbstractTreeNode;
Expand All @@ -21,11 +19,14 @@
import lombok.SneakyThrows;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;

public class BPlusTreeUniqueTreeIndexManager<K extends Comparable<K>, V> extends AbstractUniqueTreeIndexManager<K, V> {
public class BPlusTreeUniqueTreeIndexManager<K extends Comparable<K>, V> extends AbstractUniqueTreeIndexManager<K, V> implements UniqueQueryableIndex<K, V> {
private final IndexStorageManager indexStorageManager;
private final IndexIOSessionFactory indexIOSessionFactory;
private final int degree;
Expand Down Expand Up @@ -85,7 +86,7 @@ public AbstractTreeNode<K> updateIndex(K identifier, V value) throws InternalOpe
return node;
}

@Override
@Override // Todo: use BinarySearch
public Optional<V> getIndex(K identifier) throws InternalOperationException {
IndexIOSession<K> indexIOSession = this.indexIOSessionFactory.create(indexStorageManager, indexId, nodeFactory, kvSize);
AbstractLeafTreeNode<K, V> baseTreeNode = BPlusTreeUtils.getResponsibleNode(indexStorageManager, getRoot(indexIOSession), identifier, indexId, degree, nodeFactory);
Expand Down Expand Up @@ -251,4 +252,135 @@ private AbstractTreeNode<K> getRoot(IndexIOSession<K> indexIOSession) throws Int
return leafTreeNode;
}

private Iterator<V> getGreaterThanIterator(K identifier, boolean supportEQ) throws InternalOperationException {
IndexIOSession<K> indexIOSession = this.indexIOSessionFactory.create(indexStorageManager, indexId, nodeFactory, kvSize);

AbstractLeafTreeNode<K, V> leafTreeNode = BPlusTreeUtils.getResponsibleNode(indexStorageManager, getRoot(indexIOSession), identifier, indexId, degree, nodeFactory);
List<KeyValue<K, V>> keyValueList = leafTreeNode.getKeyValueList(degree);

if (keyValueList.getLast().key().compareTo(identifier) <= 0 && !supportEQ && leafTreeNode.getNextSiblingPointer(degree).isPresent()){
leafTreeNode = (AbstractLeafTreeNode<K, V>) indexIOSession.read(leafTreeNode.getNextSiblingPointer(degree).get());
keyValueList = leafTreeNode.getKeyValueList(degree);
}

final AtomicReference<AbstractLeafTreeNode<K, V>> leafReference = new AtomicReference<>(leafTreeNode);
final AtomicReference<List<KeyValue<K, V>>> keyValueListReference = new AtomicReference<>(keyValueList);

AtomicInteger index = new AtomicInteger(-1);

// Todo: binary search?
for (int i = 0; i < keyValueList.size(); i++) {
if (
supportEQ && keyValueList.get(i).key().compareTo(identifier) >= 0
||
!supportEQ && keyValueList.get(i).key().compareTo(identifier) > 0
) {
index.set(i);
break;
}
}

return new Iterator<>() {
@SneakyThrows
@Override
public boolean hasNext() {
if (index.get() == -1)
return false;

if (index.get() == keyValueListReference.get().size()) {
Optional<Pointer> nextSiblingPointerOptional = leafReference.get().getNextSiblingPointer(degree);
if (nextSiblingPointerOptional.isEmpty())
return false;
leafReference.set((AbstractLeafTreeNode<K, V>) indexIOSession.read(nextSiblingPointerOptional.get()));
keyValueListReference.set(leafReference.get().getKeyValueList(degree));
index.set(0);
}

return true;
}

@Override
public V next() {
int i = index.getAndIncrement();
return keyValueListReference.get().get(i).value();
}
};
}

private Iterator<V> getLessThanIterator(K identifier, boolean supportEQ) throws InternalOperationException {
IndexIOSession<K> indexIOSession = this.indexIOSessionFactory.create(indexStorageManager, indexId, nodeFactory, kvSize);

AbstractLeafTreeNode<K, V> leafTreeNode = BPlusTreeUtils.getResponsibleNode(indexStorageManager, getRoot(indexIOSession), identifier, indexId, degree, nodeFactory);
List<KeyValue<K, V>> keyValueList = leafTreeNode.getKeyValueList(degree);

if (keyValueList.getFirst().key().compareTo(identifier) >= 0 && !supportEQ && leafTreeNode.getPreviousSiblingPointer(degree).isPresent()){
leafTreeNode = (AbstractLeafTreeNode<K, V>) indexIOSession.read(leafTreeNode.getPreviousSiblingPointer(degree).get());
keyValueList = leafTreeNode.getKeyValueList(degree);
}

final AtomicReference<AbstractLeafTreeNode<K, V>> leafReference = new AtomicReference<>(leafTreeNode);
final AtomicReference<List<KeyValue<K, V>>> keyValueListReference = new AtomicReference<>(keyValueList);

AtomicInteger index = new AtomicInteger(-1);

// Todo: binary search?
for (int i = keyValueList.size() - 1; i >= 0; i--) {
if (
supportEQ && keyValueList.get(i).key().compareTo(identifier) <= 0
||
!supportEQ && keyValueList.get(i).key().compareTo(identifier) < 0
) {
index.set(i);
break;
}
}

return new Iterator<>() {
@SneakyThrows
@Override
public boolean hasNext() {
if (index.get() == -1) {
Optional<Pointer> siblingPointer = leafReference.get().getPreviousSiblingPointer(degree);
if (siblingPointer.isEmpty())
return false;
leafReference.set((AbstractLeafTreeNode<K, V>) indexIOSession.read(siblingPointer.get()));
List<KeyValue<K, V>> keyValueList1 = leafReference.get().getKeyValueList(degree);
keyValueListReference.set(keyValueList1);
index.set(keyValueList1.size() - 1);
}
return true;
}

@Override
public V next() {
int i = index.getAndDecrement();
return keyValueListReference.get().get(i).value();
}
};
}

@Override
public Iterator<V> getGreaterThan(K identifier) throws InternalOperationException {
return getGreaterThanIterator(identifier, false);
}

@Override
public Iterator<V> getGreaterThanEqual(K identifier) throws InternalOperationException {
return getGreaterThanIterator(identifier, true);
}

@Override
public Iterator<V> getLessThan(K k) throws InternalOperationException {
return getLessThanIterator(k, false);
}

@Override
public Iterator<V> getLessThanEqual(K k) throws InternalOperationException {
return getLessThanIterator(k, true);
}

@Override
public Optional<V> getEqual(K k) throws InternalOperationException {
return getIndex(k);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@

import com.github.sepgh.testudo.index.DuplicateIndexManager;
import com.github.sepgh.testudo.index.Pointer;
import com.github.sepgh.testudo.index.UniqueQueryableIndex;
import com.github.sepgh.testudo.index.UniqueTreeIndexManager;
import com.github.sepgh.testudo.scheme.Scheme;


public interface CollectionIndexProvider {
UniqueTreeIndexManager<?, ?> getUniqueIndexManager(Scheme.Field field);
DuplicateIndexManager<?, ?> getDuplicateIndexManager(Scheme.Field field);
UniqueQueryableIndex<?, ? extends Number> getUniqueIndexManager(Scheme.Field field);
DuplicateIndexManager<?, ? extends Number> getDuplicateIndexManager(Scheme.Field field);
UniqueTreeIndexManager<?, Pointer> getClusterIndexManager();
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.github.sepgh.testudo.operation;

import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.index.*;
import com.github.sepgh.testudo.index.data.IndexBinaryObjectFactory;
import com.github.sepgh.testudo.index.data.PointerIndexBinaryObject;
Expand All @@ -18,9 +18,9 @@

public class DefaultCollectionIndexProviderFactory implements CollectionIndexProviderFactory {
protected final Map<Scheme.Collection, CollectionIndexProvider> providers = new HashMap<>();
protected final Map<String, UniqueTreeIndexManager<?, ?>> uniqueTreeIndexManagers = new HashMap<>();
protected final Map<String, UniqueQueryableIndex<?, ? extends Number>> uniqueTreeIndexManagers = new HashMap<>();
protected final Map<String, UniqueTreeIndexManager<?, Pointer>> clusterIndexManagers = new HashMap<>();
protected final Map<String, DuplicateIndexManager<?, ?>> duplicateIndexManagers = new HashMap<>();
protected final Map<String, DuplicateIndexManager<?, ? extends Number>> duplicateIndexManagers = new HashMap<>();
protected final EngineConfig engineConfig;
protected final IndexStorageManagerFactory indexStorageManagerFactory;
protected final DatabaseStorageManager databaseStorageManager;
Expand Down Expand Up @@ -49,7 +49,7 @@ protected String getIndexId(Scheme.Collection collection, Scheme.Field field){
return "%d_%d".formatted(collection.getId(), field.getId());
}

protected UniqueTreeIndexManager<?, ?> buildUniqueIndexManager(Scheme.Collection collection, Scheme.Field field) {
protected UniqueQueryableIndex<?, ? extends Number> buildUniqueIndexManager(Scheme.Collection collection, Scheme.Field field) {
Preconditions.checkArgument(field.isPrimary() || field.isIndexUnique(), "Field should either be primary or unique to build a UniqueIndexManager");

// Raw use of field.id as indexId would force the scheme designer to use unique field ids per whole DB
Expand All @@ -60,7 +60,7 @@ protected String getIndexId(Scheme.Collection collection, Scheme.Field field){
Serializer<?> serializer = SerializerRegistry.getInstance().getSerializer(field.getType());
Serializer<?> clusterSerializer = SerializerRegistry.getInstance().getSerializer(engineConfig.getClusterKeyType().getTypeName());

return new BPlusTreeUniqueTreeIndexManager<>(
return (UniqueQueryableIndex<?, ? extends Number>) new BPlusTreeUniqueTreeIndexManager<>(
indexId,
engineConfig.getBTreeDegree(),
indexStorageManagerFactory.create(collection, field),
Expand All @@ -86,17 +86,14 @@ protected UniqueTreeIndexManager<?, Pointer> buildClusterIndexManager(Scheme.Col
protected <K extends Comparable<K>, V extends Number & Comparable<V>> DuplicateIndexManager<K, V> buildDuplicateIndexManager(Scheme.Collection collection, Scheme.Field field) {
int indexId = getIndexId(collection, field).hashCode();

UniqueTreeIndexManager<?, ?> uniqueTreeIndexManager = uniqueTreeIndexManagers.computeIfAbsent(getIndexId(collection, field), key -> {
Serializer<?> fieldSerializer = SerializerRegistry.getInstance().getSerializer(field.getType());

return new BPlusTreeUniqueTreeIndexManager<>(
indexId,
engineConfig.getBTreeDegree(),
indexStorageManagerFactory.create(collection, field),
fieldSerializer.getIndexBinaryObjectFactory(field),
new PointerIndexBinaryObject.Factory()
);
});
Serializer<?> fieldSerializer = SerializerRegistry.getInstance().getSerializer(field.getType());
BPlusTreeUniqueTreeIndexManager<?, Pointer> uniqueTreeIndexManager = new BPlusTreeUniqueTreeIndexManager<>(
indexId,
engineConfig.getBTreeDegree(),
indexStorageManagerFactory.create(collection, field),
fieldSerializer.getIndexBinaryObjectFactory(field),
new PointerIndexBinaryObject.Factory()
);

Serializer<?> clusterSerializer = SerializerRegistry.getInstance().getSerializer(engineConfig.getClusterKeyType().getTypeName());

Expand All @@ -122,7 +119,7 @@ protected CollectionIndexProvider getProvider(Scheme.Collection collection) {

return new CollectionIndexProvider() {
@Override
public UniqueTreeIndexManager<?, ?> getUniqueIndexManager(Scheme.Field field) {
public UniqueQueryableIndex<?, ? extends Number> getUniqueIndexManager(Scheme.Field field) {
Preconditions.checkNotNull(field);
Preconditions.checkArgument(field.isIndex() || field.isPrimary(), "Index Manager can only be requested for indexed fields");
Preconditions.checkArgument(field.isIndexUnique() || field.isPrimary(), "Unique index manager can only be created for fields with index set as unique or primary");
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.github.sepgh.testudo.scheme;

import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.index.KeyValue;
import com.github.sepgh.testudo.index.Pointer;
import com.github.sepgh.testudo.index.UniqueTreeIndexManager;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.github.sepgh.testudo.storage.db;

import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.exception.VerificationException;
import com.github.sepgh.testudo.index.Pointer;
import com.github.sepgh.testudo.storage.pool.FileHandlerPool;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.github.sepgh.testudo.storage.index;

import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.index.Pointer;
import com.github.sepgh.testudo.storage.index.header.IndexHeaderManager;
import com.github.sepgh.testudo.storage.index.header.IndexHeaderManagerFactory;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.github.sepgh.testudo.storage.index;

import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.index.Pointer;
import com.github.sepgh.testudo.storage.index.header.IndexHeaderManager;
import com.github.sepgh.testudo.storage.index.header.IndexHeaderManagerFactory;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.github.sepgh.testudo.storage.index;

import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.scheme.Scheme;
import com.github.sepgh.testudo.storage.index.header.IndexHeaderManagerFactory;
import com.github.sepgh.testudo.storage.pool.FileHandler;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.github.sepgh.testudo.storage.index;

import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.scheme.Scheme;
import com.github.sepgh.testudo.storage.index.header.IndexHeaderManagerFactory;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.github.sepgh.testudo.storage.index;

import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.index.Pointer;
import com.github.sepgh.testudo.storage.index.header.IndexHeaderManager;
import com.github.sepgh.testudo.storage.index.header.IndexHeaderManagerFactory;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.github.sepgh.testudo.storage.pool;

import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;

import java.io.IOException;
import java.nio.channels.AsynchronousFileChannel;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import com.github.sepgh.test.TestParams;
import com.github.sepgh.test.utils.FileUtils;
import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.index.BinaryListIterator;
import com.github.sepgh.testudo.index.data.IndexBinaryObjectFactory;
import com.github.sepgh.testudo.serialization.IntegerSerializer;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import com.github.sepgh.test.TestParams;
import com.github.sepgh.test.utils.FileUtils;
import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.index.*;
import com.github.sepgh.testudo.index.data.PointerIndexBinaryObject;
import com.github.sepgh.testudo.index.tree.BPlusTreeUniqueTreeIndexManager;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import com.github.sepgh.test.TestParams;
import com.github.sepgh.test.utils.FileUtils;
import com.github.sepgh.testudo.EngineConfig;
import com.github.sepgh.testudo.context.EngineConfig;
import com.github.sepgh.testudo.index.DuplicateBitmapIndexManager;
import com.github.sepgh.testudo.index.DuplicateIndexManager;
import com.github.sepgh.testudo.index.Pointer;
Expand Down
Loading

0 comments on commit a8c0233

Please sign in to comment.