Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Mongo #731

Merged
merged 35 commits into from
Jan 8, 2019
Merged

Mongo #731

Show file tree
Hide file tree
Changes from 30 commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
00c2ca3
Add mongo driver
Oct 2, 2018
e4564c3
Add Mongo and the factory method for it
Oct 2, 2018
38e451c
Add in the Initial Mongo Database Implementation
sdesmond46 Oct 2, 2018
d3478dd
More tests passing
sdesmond46 Oct 3, 2018
0c56b4a
Add more test configurations
Oct 3, 2018
0b70703
Add client sessions
sdesmond46 Oct 3, 2018
1b8b08e
Get aion node syncing
sdesmond46 Oct 4, 2018
2284556
Little bit of code cleanup
sdesmond46 Oct 8, 2018
cfc210c
Add authentication stuff
sdesmond46 Oct 8, 2018
c0d0ba1
Mongo tests passing (needs cleanup)
sdesmond46 Oct 9, 2018
ec9f995
Cleaning up
sdesmond46 Oct 9, 2018
cb750ff
DB tests passing
sdesmond46 Oct 9, 2018
a23d7ec
Update comments
sdesmond46 Oct 9, 2018
9194bb3
Cleanup and add shutdown hook
sdesmond46 Oct 9, 2018
e9f9d6d
Add readonly Mongo concept
sdesmond46 Oct 11, 2018
a8d16f3
Merge remote-tracking branch 'aionnetwork/master-pre-merge' into mast…
sdesmond46 Nov 16, 2018
d9705b5
Merge branch 'master-pre-merge' into mongo
sdesmond46 Nov 16, 2018
31ecbb3
Get mongo working again
sdesmond46 Nov 17, 2018
0fd3ed3
Merge remote-tracking branch 'aionnetwork/master-pre-merge' into mast…
sdesmond46 Nov 19, 2018
5ca4854
Merge branch 'master-pre-merge' into mongo
sdesmond46 Nov 19, 2018
1247d70
Get Mongo tests running again
sdesmond46 Nov 19, 2018
c29d7c0
Code review cleanup
sdesmond46 Nov 20, 2018
7f92624
Merge remote-tracking branch 'aionnetwork/master-pre-merge' into mast…
sdesmond46 Nov 21, 2018
a893c7e
Merge branch 'master-pre-merge' into mongo
sdesmond46 Nov 21, 2018
c7cdff8
Merge remote-tracking branch 'aionnetwork/master-pre-merge' into mast…
sdesmond46 Nov 27, 2018
7578059
Merge branch 'master-pre-merge' into mongo
sdesmond46 Nov 27, 2018
89d729c
Code cleanup
sdesmond46 Nov 27, 2018
bc042f9
Couple missed files
sdesmond46 Nov 27, 2018
6c37a15
Add a script to quickly run Mongo locally yourself
sdesmond46 Nov 28, 2018
4dd7695
Make test runner more reliable
sdesmond46 Nov 28, 2018
2be3a8d
Reduce log level & suppress incorrect error message (CR feedback)
sdesmond46 Dec 11, 2018
7823654
Add synchronization to MongoConnectionManager
sdesmond46 Dec 14, 2018
0e3fd7e
Switch the way we run mongo in the tests
Dec 18, 2018
f3d93a4
little bit of cleanup
Dec 18, 2018
e079cd3
Try to make Jenkins build pass
Dec 19, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added lib/mongo-java-driver-3.8.2.jar
Binary file not shown.
9 changes: 5 additions & 4 deletions modAionBase/src/org/aion/base/db/IDatabase.java
Original file line number Diff line number Diff line change
Expand Up @@ -146,12 +146,13 @@ public interface IDatabase {
boolean isAutoCommitEnabled();

/**
* Indicates if database persists to disk. Returns a value, regardless of database is opened
* (persistence vs in-memory for DB shall be configured upon instantiation of IDatabase)
* Indicates the method of persistence this database uses. Whether it's written
* to disk, only held in memory, or stored inside a database engine's proprietary
* format.
*
* @return {@code true} if data is persistent, {@code false} otherwise
* @return The method of persistence this database uses.
*/
boolean isPersistent();
PersistenceMethod getPersistenceMethod();

/**
* Used to validate if the DB file(s) has been created on disk. Can be used any time during this
Expand Down
14 changes: 14 additions & 0 deletions modAionBase/src/org/aion/base/db/PersistenceMethod.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package org.aion.base.db;

public enum PersistenceMethod {
UNKNOWN,

// The data isn't actually persisted but just stored temporarily in memory
IN_MEMORY,

// The data is stored in a file directory
FILE_BASED,

// The data is stored in the proprietary format of a database management system
DBMS
}
10 changes: 5 additions & 5 deletions modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,13 @@ protected AionRepositoryImpl(IRepositoryConfig repoConfig) {
private static class AionRepositoryImplHolder {
// configuration
private static CfgAion config = CfgAion.inst();

// repository singleton instance
private static final AionRepositoryImpl inst =
new AionRepositoryImpl(
new RepositoryConfig(
config.getDatabasePath(),
ContractDetailsAion.getInstance(),
config.getDb()));
new AionRepositoryImpl(new RepositoryConfig(
config.getDatabasePath(),
ContractDetailsAion.getInstance(),
config.getDb()));
}

public static AionRepositoryImpl inst() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ public PendingBlockStore(final Properties _props) throws InvalidFilePathExceptio

// check for database persistence requirements
DBVendor vendor = DBVendor.fromString(local.getProperty(Props.DB_TYPE));
if (vendor.getPersistence()) {
if (vendor.isFileBased()) {
File pbFolder =
new File(local.getProperty(Props.DB_PATH), local.getProperty(Props.DB_NAME));

Expand Down
11 changes: 11 additions & 0 deletions modDbImpl/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,15 @@ configurations {
testClassesOut
}


sourceSets {
test {
resources {
srcDirs = ['test_resources']
}
}
}

dependencies {
compile project(':modAionBase')
compile project(':modLogger')
Expand All @@ -12,6 +21,8 @@ dependencies {
compile group: 'org.ethereum', name: 'leveldbjni-all', version: '1.18.3'
compile group: 'org.rocksdb', name: 'rocksdbjni', version: '5.11.3'
compile group: 'com.h2database', name: 'h2-mvstore', version: '1.4.196'
compile group: 'org.mongodb', name: 'mongo-java-driver', version: '3.8.2'


testCompile 'junit:junit:4.12'
testCompile 'com.google.truth:truth:0.42'
Expand Down
1 change: 1 addition & 0 deletions modDbImpl/src/module-info.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
requires rocksdbjni;
requires h2.mvstore;
requires com.google.common;
requires mongo.java.driver;

exports org.aion.db.impl;
exports org.aion.db.impl.leveldb;
Expand Down
12 changes: 7 additions & 5 deletions modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import java.util.Optional;
import java.util.Set;
import org.aion.base.db.IByteArrayKeyValueDatabase;
import org.aion.base.db.PersistenceMethod;
import org.aion.base.util.ByteArrayWrapper;
import org.aion.db.impl.AbstractDB;
import org.aion.log.AionLoggerFactory;
Expand Down Expand Up @@ -291,8 +292,8 @@ public boolean isAutoCommitEnabled() {
}

@Override
public boolean isPersistent() {
return database.isPersistent();
public PersistenceMethod getPersistenceMethod() {
return database.getPersistenceMethod();
}

@Override
Expand Down Expand Up @@ -505,10 +506,11 @@ public void deleteBatch(Collection<byte[]> keys) {

@Override
public void drop() {
check();
if (this.isOpen()) {
this.loadingCache.invalidateAll();
this.dirtyEntries.clear();
}

this.loadingCache.invalidateAll();
this.dirtyEntries.clear();
this.database.drop();
}

Expand Down
5 changes: 3 additions & 2 deletions modDbImpl/src/org/aion/db/generic/LockedDatabase.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.aion.base.db.IByteArrayKeyValueDatabase;
import org.aion.base.db.PersistenceMethod;
import org.aion.log.AionLoggerFactory;
import org.aion.log.LogEnum;
import org.slf4j.Logger;
Expand Down Expand Up @@ -168,9 +169,9 @@ public boolean isAutoCommitEnabled() {
}

@Override
public boolean isPersistent() {
public PersistenceMethod getPersistenceMethod() {
// no locks because the persistence flag never changes
return database.isPersistent();
return database.getPersistenceMethod();
}

@Override
Expand Down
5 changes: 3 additions & 2 deletions modDbImpl/src/org/aion/db/generic/TimedDatabase.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.Optional;
import java.util.Set;
import org.aion.base.db.IByteArrayKeyValueDatabase;
import org.aion.base.db.PersistenceMethod;
import org.aion.base.util.Hex;
import org.aion.log.AionLoggerFactory;
import org.aion.log.LogEnum;
Expand Down Expand Up @@ -135,9 +136,9 @@ public boolean isAutoCommitEnabled() {
}

@Override
public boolean isPersistent() {
public PersistenceMethod getPersistenceMethod() {
// no locks because the persistence flag never changes
return database.isPersistent();
return database.getPersistenceMethod();
}

@Override
Expand Down
12 changes: 8 additions & 4 deletions modDbImpl/src/org/aion/db/impl/AbstractDB.java
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import java.util.Optional;
import java.util.stream.Stream;
import org.aion.base.db.IByteArrayKeyValueDatabase;
import org.aion.base.db.PersistenceMethod;
import org.aion.base.util.ByteArrayWrapper;
import org.aion.log.AionLoggerFactory;
import org.aion.log.LogEnum;
Expand Down Expand Up @@ -109,6 +110,7 @@ public void compact() {

@Override
public void drop() {
boolean wasOpen = isOpen();
close();

try (Stream<Path> stream = Files.walk(new File(path).toPath())) {
Expand All @@ -117,7 +119,9 @@ public void drop() {
LOG.error("Unable to delete path due to: ", e);
}

open();
if (wasOpen) {
open();
}
}

@Override
Expand Down Expand Up @@ -169,9 +173,9 @@ public boolean isAutoCommitEnabled() {
}

@Override
public boolean isPersistent() {
// always persistent when not overwritten by the class
return true;
public PersistenceMethod getPersistenceMethod() {
// Default to file-based since most of our dbs are that
return PersistenceMethod.FILE_BASED;
}

/**
Expand Down
36 changes: 24 additions & 12 deletions modDbImpl/src/org/aion/db/impl/DBVendor.java
Original file line number Diff line number Diff line change
Expand Up @@ -37,23 +37,26 @@
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.aion.base.db.PersistenceMethod;
import org.aion.db.impl.rocksdb.RocksDBWrapper;

// @ThreadSafe
public enum DBVendor {

/** Used in correlation with implementations of {@link IDriver}. */
UNKNOWN("unknown", false), //
UNKNOWN("unknown", PersistenceMethod.UNKNOWN), //
/** Using an instance of {@link org.aion.db.impl.leveldb.LevelDB}. */
LEVELDB("leveldb", true), //
LEVELDB("leveldb", PersistenceMethod.FILE_BASED), //
/** Using an instance of {@link RocksDBWrapper}. */
ROCKSDB("rocksdb", true),
ROCKSDB("rocksdb", PersistenceMethod.FILE_BASED),
/** Using an instance of {@link org.aion.db.impl.h2.H2MVMap}. */
H2("h2", true), //
H2("h2", PersistenceMethod.FILE_BASED), //
/** Using an instance of {@Link org.aion.db.impl.mongodb.MongoDB} */
MONGODB("mongodb", PersistenceMethod.DBMS),
/** Using an instance of {@link org.aion.db.impl.mockdb.MockDB}. */
MOCKDB("mockdb", false),
MOCKDB("mockdb", PersistenceMethod.IN_MEMORY),
/** Using an instance of {@link org.aion.db.impl.mockdb.PersistentMockDB}. */
PERSISTENTMOCKDB("persistentmockdb", false);
PERSISTENTMOCKDB("persistentmockdb", PersistenceMethod.FILE_BASED);

private static final Map<String, DBVendor> stringToTypeMap = new ConcurrentHashMap<>();

Expand All @@ -65,12 +68,12 @@ public enum DBVendor {

/* map implemented using concurrent hash map */
private static final List<DBVendor> driverImplementations =
List.of(LEVELDB, ROCKSDB, H2, MOCKDB);
List.of(LEVELDB, ROCKSDB, H2, MOCKDB, MONGODB);

private final String value;
private final boolean persistence;
private final PersistenceMethod persistence;

DBVendor(final String value, final boolean persistent) {
DBVendor(final String value, final PersistenceMethod persistent) {
this.value = value;
this.persistence = persistent;
}
Expand All @@ -94,14 +97,23 @@ public String toValue() {
}

/**
* Check whether the DB provided by the vendor is intended to be persistent.
* Gets the persistence method of this database vendor
*
* @return {@code true} if the DB provider is intended to be persistent
* @return The persistence method of the database
*/
public boolean getPersistence() {
public PersistenceMethod getPersistence() {
return this.persistence;
}

/**
* Gets Whether or not this database uses file-based persistence
*
* @return Whether or not this database uses file-based persistence
*/
public boolean isFileBased() {
return this.persistence == PersistenceMethod.FILE_BASED;
}

/** @return {@code false} for a DBVendor with an undefined driver implementation */
public static boolean hasDriverImplementation(DBVendor v) {
return driverImplementations.contains(v);
Expand Down
5 changes: 5 additions & 0 deletions modDbImpl/src/org/aion/db/impl/DatabaseFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.aion.db.impl.leveldb.LevelDB;
import org.aion.db.impl.leveldb.LevelDBConstants;
import org.aion.db.impl.mockdb.MockDB;
import org.aion.db.impl.mongodb.MongoDB;
import org.aion.db.impl.mockdb.PersistentMockDB;
import org.aion.db.impl.rocksdb.RocksDBConstants;
import org.aion.db.impl.rocksdb.RocksDBWrapper;
Expand Down Expand Up @@ -214,6 +215,10 @@ private static AbstractDB connectBasic(Properties info) {
{
return new H2MVMap(dbName, dbPath, enableDbCache, enableDbCompression);
}
case MONGODB:
{
return new MongoDB(dbName, dbPath);
}
default:
break;
}
Expand Down
6 changes: 4 additions & 2 deletions modDbImpl/src/org/aion/db/impl/mockdb/MockDB.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.aion.base.db.PersistenceMethod;
import org.aion.base.util.ByteArrayWrapper;
import org.aion.db.impl.AbstractDB;

Expand Down Expand Up @@ -80,8 +81,9 @@ public boolean isOpen() {
}

@Override
public boolean isPersistent() {
return false;
public PersistenceMethod getPersistenceMethod() {
// MockDB doesn't persist anything to disk, so it's type is IN_MEMORY
return PersistenceMethod.IN_MEMORY;
}

@Override
Expand Down
6 changes: 4 additions & 2 deletions modDbImpl/src/org/aion/db/impl/mockdb/PersistentMockDB.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import org.aion.base.db.PersistenceMethod;
import org.aion.base.util.ByteArrayWrapper;

/**
Expand Down Expand Up @@ -111,8 +112,9 @@ private static final byte[] convertToByteArray(String byteArrayString) {
* open and saved to disk at close.
*/
@Override
public boolean isPersistent() {
return true;
public PersistenceMethod getPersistenceMethod() {
// return file-based so file cleanup / setup can proceed as expected
return PersistenceMethod.FILE_BASED;
}

/** @implNote Returns false because data is saved to disk only at close. */
Expand Down
Loading