diff --git a/Jenkinsfile b/Jenkinsfile index 350c622725..c30f574288 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -73,4 +73,5 @@ environment { GIT_BRANCH = "${env.BRANCH_NAME}" + DOCKER_HOST = "unix:///var/run/docker.sock" } diff --git a/lib/mongo-java-driver-3.8.2.jar b/lib/mongo-java-driver-3.8.2.jar new file mode 100644 index 0000000000..2bb864f178 Binary files /dev/null and b/lib/mongo-java-driver-3.8.2.jar differ diff --git a/modAionBase/src/org/aion/base/db/IDatabase.java b/modAionBase/src/org/aion/base/db/IDatabase.java index 4d93eb6d88..27b50c24f6 100644 --- a/modAionBase/src/org/aion/base/db/IDatabase.java +++ b/modAionBase/src/org/aion/base/db/IDatabase.java @@ -146,12 +146,13 @@ public interface IDatabase { boolean isAutoCommitEnabled(); /** - * Indicates if database persists to disk. Returns a value, regardless of database is opened - * (persistence vs in-memory for DB shall be configured upon instantiation of IDatabase) + * Indicates the method of persistence this database uses. Whether it's written + * to disk, only held in memory, or stored inside a database engine's proprietary + * format. * - * @return {@code true} if data is persistent, {@code false} otherwise + * @return The method of persistence this database uses. */ - boolean isPersistent(); + PersistenceMethod getPersistenceMethod(); /** * Used to validate if the DB file(s) has been created on disk. Can be used any time during this diff --git a/modAionBase/src/org/aion/base/db/PersistenceMethod.java b/modAionBase/src/org/aion/base/db/PersistenceMethod.java new file mode 100644 index 0000000000..9c0f2cefe8 --- /dev/null +++ b/modAionBase/src/org/aion/base/db/PersistenceMethod.java @@ -0,0 +1,14 @@ +package org.aion.base.db; + +public enum PersistenceMethod { + UNKNOWN, + + // The data isn't actually persisted but just stored temporarily in memory + IN_MEMORY, + + // The data is stored in a file directory + FILE_BASED, + + // The data is stored in the proprietary format of a database management system + DBMS +} diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java index 317c8c3398..588493d06b 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java @@ -82,13 +82,13 @@ protected AionRepositoryImpl(IRepositoryConfig repoConfig) { private static class AionRepositoryImplHolder { // configuration private static CfgAion config = CfgAion.inst(); + // repository singleton instance private static final AionRepositoryImpl inst = - new AionRepositoryImpl( - new RepositoryConfig( - config.getDatabasePath(), - ContractDetailsAion.getInstance(), - config.getDb())); + new AionRepositoryImpl(new RepositoryConfig( + config.getDatabasePath(), + ContractDetailsAion.getInstance(), + config.getDb())); } public static AionRepositoryImpl inst() { diff --git a/modAionImpl/src/org/aion/zero/impl/db/PendingBlockStore.java b/modAionImpl/src/org/aion/zero/impl/db/PendingBlockStore.java index 932309497a..45b8b958bf 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/PendingBlockStore.java +++ b/modAionImpl/src/org/aion/zero/impl/db/PendingBlockStore.java @@ -129,7 +129,7 @@ public PendingBlockStore(final Properties _props) throws InvalidFilePathExceptio // check for database persistence requirements DBVendor vendor = DBVendor.fromString(local.getProperty(Props.DB_TYPE)); - if (vendor.getPersistence()) { + if (vendor.isFileBased()) { File pbFolder = new File(local.getProperty(Props.DB_PATH), local.getProperty(Props.DB_NAME)); diff --git a/modDbImpl/build.gradle b/modDbImpl/build.gradle index b816082ecf..3053406d5b 100644 --- a/modDbImpl/build.gradle +++ b/modDbImpl/build.gradle @@ -4,6 +4,15 @@ configurations { testClassesOut } + +sourceSets { + test { + resources { + srcDirs = ['test_resources'] + } + } +} + dependencies { compile project(':modAionBase') compile project(':modLogger') @@ -12,12 +21,15 @@ dependencies { compile group: 'org.ethereum', name: 'leveldbjni-all', version: '1.18.3' compile group: 'org.rocksdb', name: 'rocksdbjni', version: '5.11.3' compile group: 'com.h2database', name: 'h2-mvstore', version: '1.4.196' + compile group: 'org.mongodb', name: 'mongo-java-driver', version: '3.8.2' + testCompile 'junit:junit:4.12' testCompile 'com.google.truth:truth:0.42' testCompile 'org.hamcrest:hamcrest-core:1.3' testCompile "org.mockito:mockito-core:2.23.0" testCompile 'pl.pragmatists:JUnitParams:1.1.1' + testCompile 'com.spotify:docker-client:8.14.5' // modAionImpl's tests uses test util classes from // this module's test sources. Gradle Java plug-in diff --git a/modDbImpl/src/module-info.java b/modDbImpl/src/module-info.java index 23cde57f35..4569a0f7ad 100644 --- a/modDbImpl/src/module-info.java +++ b/modDbImpl/src/module-info.java @@ -6,6 +6,7 @@ requires rocksdbjni; requires h2.mvstore; requires com.google.common; + requires mongo.java.driver; exports org.aion.db.impl; exports org.aion.db.impl.leveldb; diff --git a/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java b/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java index 8aa1fe8f2f..08e5fc6d1d 100644 --- a/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java +++ b/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java @@ -46,6 +46,7 @@ import java.util.Optional; import java.util.Set; import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.db.PersistenceMethod; import org.aion.base.util.ByteArrayWrapper; import org.aion.db.impl.AbstractDB; import org.aion.log.AionLoggerFactory; @@ -291,8 +292,8 @@ public boolean isAutoCommitEnabled() { } @Override - public boolean isPersistent() { - return database.isPersistent(); + public PersistenceMethod getPersistenceMethod() { + return database.getPersistenceMethod(); } @Override @@ -505,10 +506,11 @@ public void deleteBatch(Collection keys) { @Override public void drop() { - check(); + if (this.isOpen()) { + this.loadingCache.invalidateAll(); + this.dirtyEntries.clear(); + } - this.loadingCache.invalidateAll(); - this.dirtyEntries.clear(); this.database.drop(); } diff --git a/modDbImpl/src/org/aion/db/generic/LockedDatabase.java b/modDbImpl/src/org/aion/db/generic/LockedDatabase.java index c46f71bda8..ef7c43d4a5 100644 --- a/modDbImpl/src/org/aion/db/generic/LockedDatabase.java +++ b/modDbImpl/src/org/aion/db/generic/LockedDatabase.java @@ -30,6 +30,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.db.PersistenceMethod; import org.aion.log.AionLoggerFactory; import org.aion.log.LogEnum; import org.slf4j.Logger; @@ -168,9 +169,9 @@ public boolean isAutoCommitEnabled() { } @Override - public boolean isPersistent() { + public PersistenceMethod getPersistenceMethod() { // no locks because the persistence flag never changes - return database.isPersistent(); + return database.getPersistenceMethod(); } @Override diff --git a/modDbImpl/src/org/aion/db/generic/TimedDatabase.java b/modDbImpl/src/org/aion/db/generic/TimedDatabase.java index 94c6621dd4..c05923b32e 100644 --- a/modDbImpl/src/org/aion/db/generic/TimedDatabase.java +++ b/modDbImpl/src/org/aion/db/generic/TimedDatabase.java @@ -28,6 +28,7 @@ import java.util.Optional; import java.util.Set; import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.db.PersistenceMethod; import org.aion.base.util.Hex; import org.aion.log.AionLoggerFactory; import org.aion.log.LogEnum; @@ -135,9 +136,9 @@ public boolean isAutoCommitEnabled() { } @Override - public boolean isPersistent() { + public PersistenceMethod getPersistenceMethod() { // no locks because the persistence flag never changes - return database.isPersistent(); + return database.getPersistenceMethod(); } @Override diff --git a/modDbImpl/src/org/aion/db/impl/AbstractDB.java b/modDbImpl/src/org/aion/db/impl/AbstractDB.java index cbf2f611bf..b0f711f13e 100644 --- a/modDbImpl/src/org/aion/db/impl/AbstractDB.java +++ b/modDbImpl/src/org/aion/db/impl/AbstractDB.java @@ -44,6 +44,7 @@ import java.util.Optional; import java.util.stream.Stream; import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.db.PersistenceMethod; import org.aion.base.util.ByteArrayWrapper; import org.aion.log.AionLoggerFactory; import org.aion.log.LogEnum; @@ -109,6 +110,7 @@ public void compact() { @Override public void drop() { + boolean wasOpen = isOpen(); close(); try (Stream stream = Files.walk(new File(path).toPath())) { @@ -117,7 +119,9 @@ public void drop() { LOG.error("Unable to delete path due to: ", e); } - open(); + if (wasOpen) { + open(); + } } @Override @@ -169,9 +173,9 @@ public boolean isAutoCommitEnabled() { } @Override - public boolean isPersistent() { - // always persistent when not overwritten by the class - return true; + public PersistenceMethod getPersistenceMethod() { + // Default to file-based since most of our dbs are that + return PersistenceMethod.FILE_BASED; } /** diff --git a/modDbImpl/src/org/aion/db/impl/DBVendor.java b/modDbImpl/src/org/aion/db/impl/DBVendor.java index db85a4a4aa..c545b5f960 100644 --- a/modDbImpl/src/org/aion/db/impl/DBVendor.java +++ b/modDbImpl/src/org/aion/db/impl/DBVendor.java @@ -37,23 +37,26 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.aion.base.db.PersistenceMethod; import org.aion.db.impl.rocksdb.RocksDBWrapper; // @ThreadSafe public enum DBVendor { /** Used in correlation with implementations of {@link IDriver}. */ - UNKNOWN("unknown", false), // + UNKNOWN("unknown", PersistenceMethod.UNKNOWN), // /** Using an instance of {@link org.aion.db.impl.leveldb.LevelDB}. */ - LEVELDB("leveldb", true), // + LEVELDB("leveldb", PersistenceMethod.FILE_BASED), // /** Using an instance of {@link RocksDBWrapper}. */ - ROCKSDB("rocksdb", true), + ROCKSDB("rocksdb", PersistenceMethod.FILE_BASED), /** Using an instance of {@link org.aion.db.impl.h2.H2MVMap}. */ - H2("h2", true), // + H2("h2", PersistenceMethod.FILE_BASED), // + /** Using an instance of {@Link org.aion.db.impl.mongodb.MongoDB} */ + MONGODB("mongodb", PersistenceMethod.DBMS), /** Using an instance of {@link org.aion.db.impl.mockdb.MockDB}. */ - MOCKDB("mockdb", false), + MOCKDB("mockdb", PersistenceMethod.IN_MEMORY), /** Using an instance of {@link org.aion.db.impl.mockdb.PersistentMockDB}. */ - PERSISTENTMOCKDB("persistentmockdb", false); + PERSISTENTMOCKDB("persistentmockdb", PersistenceMethod.FILE_BASED); private static final Map stringToTypeMap = new ConcurrentHashMap<>(); @@ -65,12 +68,12 @@ public enum DBVendor { /* map implemented using concurrent hash map */ private static final List driverImplementations = - List.of(LEVELDB, ROCKSDB, H2, MOCKDB); + List.of(LEVELDB, ROCKSDB, H2, MOCKDB, MONGODB); private final String value; - private final boolean persistence; + private final PersistenceMethod persistence; - DBVendor(final String value, final boolean persistent) { + DBVendor(final String value, final PersistenceMethod persistent) { this.value = value; this.persistence = persistent; } @@ -94,14 +97,23 @@ public String toValue() { } /** - * Check whether the DB provided by the vendor is intended to be persistent. + * Gets the persistence method of this database vendor * - * @return {@code true} if the DB provider is intended to be persistent + * @return The persistence method of the database */ - public boolean getPersistence() { + public PersistenceMethod getPersistence() { return this.persistence; } + /** + * Gets Whether or not this database uses file-based persistence + * + * @return Whether or not this database uses file-based persistence + */ + public boolean isFileBased() { + return this.persistence == PersistenceMethod.FILE_BASED; + } + /** @return {@code false} for a DBVendor with an undefined driver implementation */ public static boolean hasDriverImplementation(DBVendor v) { return driverImplementations.contains(v); diff --git a/modDbImpl/src/org/aion/db/impl/DatabaseFactory.java b/modDbImpl/src/org/aion/db/impl/DatabaseFactory.java index 67b8e6ab2e..fcaf0b9949 100644 --- a/modDbImpl/src/org/aion/db/impl/DatabaseFactory.java +++ b/modDbImpl/src/org/aion/db/impl/DatabaseFactory.java @@ -33,6 +33,7 @@ import org.aion.db.impl.leveldb.LevelDB; import org.aion.db.impl.leveldb.LevelDBConstants; import org.aion.db.impl.mockdb.MockDB; +import org.aion.db.impl.mongodb.MongoDB; import org.aion.db.impl.mockdb.PersistentMockDB; import org.aion.db.impl.rocksdb.RocksDBConstants; import org.aion.db.impl.rocksdb.RocksDBWrapper; @@ -214,6 +215,10 @@ private static AbstractDB connectBasic(Properties info) { { return new H2MVMap(dbName, dbPath, enableDbCache, enableDbCompression); } + case MONGODB: + { + return new MongoDB(dbName, dbPath); + } default: break; } diff --git a/modDbImpl/src/org/aion/db/impl/mockdb/MockDB.java b/modDbImpl/src/org/aion/db/impl/mockdb/MockDB.java index 0d6c96eea5..70d336b782 100644 --- a/modDbImpl/src/org/aion/db/impl/mockdb/MockDB.java +++ b/modDbImpl/src/org/aion/db/impl/mockdb/MockDB.java @@ -28,6 +28,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.aion.base.db.PersistenceMethod; import org.aion.base.util.ByteArrayWrapper; import org.aion.db.impl.AbstractDB; @@ -80,8 +81,9 @@ public boolean isOpen() { } @Override - public boolean isPersistent() { - return false; + public PersistenceMethod getPersistenceMethod() { + // MockDB doesn't persist anything to disk, so it's type is IN_MEMORY + return PersistenceMethod.IN_MEMORY; } @Override diff --git a/modDbImpl/src/org/aion/db/impl/mockdb/PersistentMockDB.java b/modDbImpl/src/org/aion/db/impl/mockdb/PersistentMockDB.java index f7940822e8..21d716f7ce 100644 --- a/modDbImpl/src/org/aion/db/impl/mockdb/PersistentMockDB.java +++ b/modDbImpl/src/org/aion/db/impl/mockdb/PersistentMockDB.java @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; +import org.aion.base.db.PersistenceMethod; import org.aion.base.util.ByteArrayWrapper; /** @@ -111,8 +112,9 @@ private static final byte[] convertToByteArray(String byteArrayString) { * open and saved to disk at close. */ @Override - public boolean isPersistent() { - return true; + public PersistenceMethod getPersistenceMethod() { + // return file-based so file cleanup / setup can proceed as expected + return PersistenceMethod.FILE_BASED; } /** @implNote Returns false because data is saved to disk only at close. */ diff --git a/modDbImpl/src/org/aion/db/impl/mongodb/MongoConnectionManager.java b/modDbImpl/src/org/aion/db/impl/mongodb/MongoConnectionManager.java new file mode 100644 index 0000000000..0ceb570e2e --- /dev/null +++ b/modDbImpl/src/org/aion/db/impl/mongodb/MongoConnectionManager.java @@ -0,0 +1,66 @@ +package org.aion.db.impl.mongodb; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import org.aion.log.AionLoggerFactory; +import org.aion.log.LogEnum; +import org.slf4j.Logger; + +/** + * This class exists to manage singleton instances to a MongoDB server. It is recommended by the Mongo + * docs to only have a single instance of the {@link com.mongodb.MongoClient} opened at a time, so this + * class keeps track of reference counting active instances and will close the connection once all instances + * are done being used. + */ +public class MongoConnectionManager { + protected static final Logger LOG = AionLoggerFactory.getLogger(LogEnum.DB.name()); + + private MongoConnectionManager() { + // Private constructor to force using the Singleton + } + + private static class Holder { + static final MongoConnectionManager INSTANCE = new MongoConnectionManager(); + } + + public static MongoConnectionManager inst() { + return Holder.INSTANCE; + } + + private Map mongoUriToClientMap = new ConcurrentHashMap<>(); + private Map activeClientCountMap = new ConcurrentHashMap<>(); + + public synchronized MongoClient getMongoClientInstance(String mongoClientUri) { + MongoClient mongoClient; + if (!this.mongoUriToClientMap.containsKey(mongoClientUri)) { + LOG.info("Creating new mongo client to connect to {}", mongoClientUri); + mongoClient = MongoClients.create(mongoClientUri); + this.mongoUriToClientMap.put(mongoClientUri, mongoClient); + this.activeClientCountMap.put(mongoClientUri, new AtomicInteger(1)); + } else { + LOG.info("Reusing existing mongo client for {}", mongoClientUri); + mongoClient = this.mongoUriToClientMap.get(mongoClientUri); + this.activeClientCountMap.get(mongoClientUri).incrementAndGet(); + } + + return mongoClient; + } + + public synchronized void closeMongoClientInstance(String mongoClientUri) { + if (!this.mongoUriToClientMap.containsKey(mongoClientUri) || !this.activeClientCountMap.containsKey(mongoClientUri)) { + throw new IllegalArgumentException(String.format("Unopened client uri %s", mongoClientUri)); + } + + int newCount = this.activeClientCountMap.get(mongoClientUri).decrementAndGet(); + if (newCount == 0) { + LOG.info("Closing mongo client connection for {}", mongoClientUri); + + this.mongoUriToClientMap.get(mongoClientUri).close(); + this.mongoUriToClientMap.remove(mongoClientUri); + this.activeClientCountMap.remove(mongoClientUri); + } + } +} diff --git a/modDbImpl/src/org/aion/db/impl/mongodb/MongoConstants.java b/modDbImpl/src/org/aion/db/impl/mongodb/MongoConstants.java new file mode 100644 index 0000000000..4678846594 --- /dev/null +++ b/modDbImpl/src/org/aion/db/impl/mongodb/MongoConstants.java @@ -0,0 +1,9 @@ +package org.aion.db.impl.mongodb; + +public class MongoConstants { + private MongoConstants() { } + + public static String AION_DB_NAME = "aion"; + public static String ID_FIELD_NAME = "_id"; + public static String VALUE_FIELD_NAME = "value"; +} diff --git a/modDbImpl/src/org/aion/db/impl/mongodb/MongoDB.java b/modDbImpl/src/org/aion/db/impl/mongodb/MongoDB.java new file mode 100644 index 0000000000..c30245590c --- /dev/null +++ b/modDbImpl/src/org/aion/db/impl/mongodb/MongoDB.java @@ -0,0 +1,425 @@ +package org.aion.db.impl.mongodb; + +import static com.mongodb.client.model.Filters.eq; + +import com.mongodb.ClientSessionOptions; +import com.mongodb.ReadConcern; +import com.mongodb.ReadPreference; +import com.mongodb.TransactionOptions; +import com.mongodb.WriteConcern; +import com.mongodb.bulk.BulkWriteResult; +import com.mongodb.client.ClientSession; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.DeleteOneModel; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import com.mongodb.client.model.UpdateOptions; +import com.mongodb.client.model.Updates; +import com.mongodb.client.model.WriteModel; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.aion.base.db.PersistenceMethod; +import org.aion.base.util.ByteArrayWrapper; +import org.aion.db.impl.AbstractDB; +import org.bson.BsonBinary; +import org.bson.BsonDocument; + +/** + * This class allows us to connect to a MongoDB instance to write the kernel's data. To test this out locally, you can + * use the script located at modDbImpl/test_resources/mongo/start_mongo_local.sh to spin up a mongo database using + * docker, and then modify your config.xml to point to that local database + */ +public class MongoDB extends AbstractDB { + + /** + * Simple wrapper class to encloses a collection of writes (inserts, edits, or deletes) to the Mongo + * database. + */ + private static class WriteBatch { + private List> edits = new ArrayList<>(); + + /** + * Adds a new edit to the batch + * @param key the key to write + * @param value the value to write. Null indicates we should delete this key + * @return this + */ + public WriteBatch addEdit(byte[] key, byte[] value) { + if (value == null) { + DeleteOneModel deleteModel = new DeleteOneModel<>( + eq(MongoConstants.ID_FIELD_NAME, new BsonBinary(key)) + ); + + edits.add(deleteModel); + } else { + UpdateOneModel updateModel = new UpdateOneModel<>( + eq(MongoConstants.ID_FIELD_NAME, new BsonBinary(key)), + Updates.set(MongoConstants.VALUE_FIELD_NAME, new BsonBinary(value)), + new UpdateOptions().upsert(true)); + + edits.add(updateModel); + } + + return this; + } + + /** + * Adds a new edit to the batch + * @param key the key to write + * @param value the value to write. Null indicates we should delete this key + * @return this + */ + public WriteBatch addEdit(ByteArrayWrapper key, byte[] value) { + return addEdit(key.getData(), value); + } + + /** + * Adds a collection of edits to the batch + * @param kvPairs The collection of key value pairs we want to write in + * @return this + */ + public WriteBatch addEdits(Map kvPairs) { + for (byte[] key : kvPairs.keySet()) { + addEdit(key, kvPairs.get(key)); + } + + return this; + } + + /** + * Adds a collection of edits to the batch + * @param kvPairs The collection of key value pairs we want to write in + * @return this + */ + public WriteBatch addEditsWrapper(Map kvPairs) { + for (ByteArrayWrapper key : kvPairs.keySet()) { + addEdit(key, kvPairs.get(key)); + } + + return this; + } + + /** + * Gets the collection of writes which have been collected here + * @return The edits which have been added to this instance. + */ + public List> getEdits() { + return this.edits; + } + + /** + * Gets the number of deletes which are in this batch + * @return Number of deletes + */ + public long getDeleteCount() { + return this.edits.stream().filter(e -> e instanceof DeleteOneModel).count(); + } + + /** + * Gets the number of updates (edits or inserts) in this batch + * @return Number of updates + */ + public long getUpdateCount() { + return this.edits.stream().filter(e -> e instanceof UpdateOneModel).count(); + } + } + + /** + * Wrapper class holding the result of writing a batch + */ + private static class WriteBatchResult { + + /** + * Total number of updates which were committed + */ + public final long totalUpdates; + + /** + * Total number of deletes which were committed + */ + public final long totalDeletes; + + /** + * Whether or not our database is read-only, which means we'll never have written anything + */ + public final boolean isReadOnly; + + /** + * Creates a new instance of the WriteBatchResult from Mongo's raw BulkWriteResult + * @param writeResult The BulkWriteResult returned from Mongo + */ + public WriteBatchResult(BulkWriteResult writeResult) { + this.totalUpdates = writeResult.getInsertedCount() + writeResult.getModifiedCount() + writeResult.getUpserts().size(); + this.totalDeletes = writeResult.getDeletedCount(); + this.isReadOnly = false; + } + + /** + * Overloaded constructor to return a dummy WriteBatchResult if we're ready only. + * @param isReadOnly Whether or not our database is read only + */ + public WriteBatchResult(boolean isReadOnly) { + this.totalUpdates = 0; + this.totalDeletes = 0; + this.isReadOnly = isReadOnly; + } + + /** + * Returns whether or not the expeced number of updates and deletes where committed in this batch + * @param batch The batch which specified these results + * @return Whether or not things were written as expected + */ + public boolean matchedExpectation(WriteBatch batch) { + return (batch.getDeleteCount() == this.totalDeletes && batch.getUpdateCount() == this.totalUpdates) || + isReadOnly; + } + } + + + private String mongoClientUri; + private ClientSession clientSession; + private MongoCollection collection = null; + private WriteBatch batch = null; + private boolean isReadOnly; + + public MongoDB(String dbName, String mongoClientUri) { + super(dbName); + this.mongoClientUri = mongoClientUri; + + this.isReadOnly = mongoClientUri.contains("reader"); + } + + /** + * Private helper method for writing a collection of edits into the database + * @param edits The edits to write + * @return A summary of the write results + */ + private WriteBatchResult doBulkWrite(WriteBatch edits) { + if (this.isReadOnly) { + LOG.info("Skipping writing because database is read only"); + return new WriteBatchResult(true); + } + + BulkWriteResult writeResult = this.collection.bulkWrite(this.clientSession, edits.getEdits()); + WriteBatchResult result = new WriteBatchResult(writeResult); + + if (result.totalDeletes != edits.getDeleteCount()) { + LOG.debug("Expected {} deletes but only deleted {}", edits.getDeleteCount(), result.totalDeletes); + } + + if (result.totalUpdates != edits.getUpdateCount()) { + LOG.debug("Expected {} upserts but only got {}", edits.getUpdateCount(), result.totalUpdates); + } + + LOG.debug("Successfully wrote {} edits", edits.getEdits().size()); + + return result; + } + + @Override + public boolean open() { + if (isOpen()) { + return true; + } + + LOG.info("Initializing MongoDB at {}", mongoClientUri); + + // Get the client and create a session for this instance + MongoClient mongoClient = MongoConnectionManager.inst().getMongoClientInstance(this.mongoClientUri); + ClientSessionOptions sessionOptions = ClientSessionOptions.builder() + .causallyConsistent(true) + .defaultTransactionOptions(TransactionOptions.builder() + .readConcern(ReadConcern.DEFAULT) + .writeConcern(WriteConcern.MAJORITY) + .readPreference(ReadPreference.nearest()) + .build()) + .build(); + this.clientSession = mongoClient.startSession(sessionOptions); + + // Get the database and our collection. Mongo takes care of creating these if they don't exist + MongoDatabase mongoDb = mongoClient.getDatabase(MongoConstants.AION_DB_NAME); + + // Gets the collection where we will be saving our values. Mongo creates it if it doesn't yet exist + this.collection = mongoDb.getCollection(this.name, BsonDocument.class); + + LOG.info("Finished opening the Mongo connection"); + return isOpen(); + } + + @Override + public boolean isOpen() { + return this.collection != null; + } + + @Override + public boolean isCreatedOnDisk() { + // Always return false here since we don't persist to the local disk. + return false; + } + + @Override + public long approximateSize() { + check(); + + // Just return -1 because we don't have a good way of asking the Mongo Server our size in bytes + return -1L; + } + + @Override + public boolean isEmpty() { + check(); + long count = this.collection.estimatedDocumentCount(); + LOG.info("Estimated document count: {}", count); + return count == 0; + } + + @Override + public Set keys() { + check(); + + LOG.debug("Getting the collection of keys"); + + Set keys = this.collection.find(this.clientSession) + .projection(Projections.fields(Projections.include(MongoConstants.ID_FIELD_NAME))) + .map(f -> f.getBinary(MongoConstants.ID_FIELD_NAME).getData()) + .into(new HashSet<>()); + + LOG.debug("The database contains {} keys", keys.size()); + + return keys; + } + + @Override + public boolean commitCache(Map cache) { + check(); + check(cache.keySet().stream().map(k -> k.getData()).collect(Collectors.toList())); + + WriteBatch edits = new WriteBatch().addEditsWrapper(cache); + WriteBatchResult result = doBulkWrite(edits); + + return result.matchedExpectation(edits); + } + + @Override + protected byte[] getInternal(byte[] k) { + BsonDocument document = this.collection.find(this.clientSession, eq(MongoConstants.ID_FIELD_NAME, new BsonBinary(k))).first(); + if (document == null) { + return null; + } else { + return document.getBinary(MongoConstants.VALUE_FIELD_NAME).getData(); + } + } + + @Override + public void put(byte[] key, byte[] value) { + check(); + check(key); + + // Write this single edit in as a batch + WriteBatch edits = new WriteBatch().addEdit(key, value); + doBulkWrite(edits); + } + + @Override + public void delete(byte[] key) { + check(); + check(key); + + // Write this single edit in as a batch + WriteBatch edits = new WriteBatch().addEdit(key, null); + doBulkWrite(edits); + } + + @Override + public void putBatch(Map inputMap) { + check(); + check(inputMap.keySet()); + + WriteBatch edits = new WriteBatch().addEdits(inputMap); + doBulkWrite(edits); + } + + @Override + public void putToBatch(byte[] key, byte[] value) { + check(); + check(key); + + if (this.batch == null) { + this.batch = new WriteBatch(); + } + + batch.addEdit(key, value); + } + + @Override + public void commitBatch() { + check(); + + if (this.batch != null) { + LOG.debug("Committing batch of writes"); + doBulkWrite(this.batch); + } else { + LOG.debug("Attempting to commit empty batch, skipping"); + } + + this.batch = null; + } + + @Override + public void deleteBatch(Collection keys) { + check(); + check(keys); + + if (!keys.isEmpty()) { + Map batch = new HashMap(); + keys.forEach(key -> batch.put(key, null)); + this.putBatch(batch); + } + } + + @Override + public void close() { + // do nothing if already closed + if (collection == null) { + return; + } + + LOG.info("Closing database " + this.toString()); + + MongoConnectionManager.inst().closeMongoClientInstance(this.mongoClientUri); + this.collection = null; + this.clientSession = null; + this.batch = null; + } + + @Override + public void drop() { + check(); + + if (this.isReadOnly) { + LOG.info("read-only database. Not dropping."); + return; + } + + LOG.info("Dropping collection {}", this.name); + this.collection.drop(this.clientSession); + } + + @Override + public PersistenceMethod getPersistenceMethod() { + return PersistenceMethod.DBMS; + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + ":" + propertiesInfo(); + } +} diff --git a/modDbImpl/test/org/aion/db/impl/DatabaseTestUtils.java b/modDbImpl/test/org/aion/db/impl/DatabaseTestUtils.java index 52b8e138b9..6726786c49 100644 --- a/modDbImpl/test/org/aion/db/impl/DatabaseTestUtils.java +++ b/modDbImpl/test/org/aion/db/impl/DatabaseTestUtils.java @@ -25,8 +25,12 @@ import static org.aion.db.impl.DatabaseFactory.Props; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.File; +import java.io.IOException; +import java.net.ServerSocket; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -37,6 +41,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; + +import org.aion.db.utils.MongoTestRunner; import org.aion.db.impl.leveldb.LevelDBConstants; public class DatabaseTestUtils { @@ -46,7 +52,7 @@ public class DatabaseTestUtils { private static final String dbPath = testDir.getAbsolutePath(); private static final Set sizeHeapCache = Set.of("0", "256"); private static final Set vendors = - Set.of(DBVendor.MOCKDB, DBVendor.H2, DBVendor.LEVELDB, DBVendor.ROCKSDB); + Set.of(DBVendor.MOCKDB, DBVendor.H2, DBVendor.LEVELDB, DBVendor.ROCKSDB, DBVendor.MONGODB); private static final String enabled = String.valueOf(Boolean.TRUE); private static final String disabled = String.valueOf(Boolean.FALSE); private static final Set options = Set.of(enabled, disabled); @@ -134,6 +140,12 @@ public static Object databaseInstanceDefinitions() { private static void addDatabaseWithCacheAndCompression( DBVendor vendor, Properties sharedProps, List parameters) { + + if (vendor == DBVendor.MONGODB) { + sharedProps = (Properties)sharedProps.clone(); + sharedProps.setProperty(Props.DB_PATH, MongoTestRunner.inst().getConnectionString()); + } + if (vendor != DBVendor.MOCKDB) { // enable/disable db_cache for (String db_cache : options) { @@ -210,4 +222,18 @@ public static void assertConcurrent( message + "failed with " + exceptions.size() + " exception(s):" + exceptions, exceptions.isEmpty()); } + + /** + * Helper method to find an unused port of the local machine + * @return An unused port + */ + public static int findOpenPort() { + try (ServerSocket socket = new ServerSocket(0);) { + return socket.getLocalPort(); + } catch (Exception ex) { + fail("Exception thrown finding open port: " + ex .getMessage()); + } + + return -1; + } } diff --git a/modDbImpl/test/org/aion/db/impl/DriverBaseTest.java b/modDbImpl/test/org/aion/db/impl/DriverBaseTest.java index 307bf376a6..42582035b9 100644 --- a/modDbImpl/test/org/aion/db/impl/DriverBaseTest.java +++ b/modDbImpl/test/org/aion/db/impl/DriverBaseTest.java @@ -46,13 +46,16 @@ import java.util.Map; import java.util.Set; import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.db.PersistenceMethod; import org.aion.db.generic.DatabaseWithCache; import org.aion.db.generic.LockedDatabase; import org.aion.db.impl.h2.H2MVMap; import org.aion.db.impl.leveldb.LevelDB; import org.aion.db.impl.mockdb.MockDB; import org.aion.db.impl.mockdb.PersistentMockDB; +import org.aion.db.impl.mongodb.MongoDB; import org.aion.db.utils.FileUtils; +import org.aion.db.utils.MongoTestRunner; import org.aion.log.AionLoggerFactory; import org.junit.After; import org.junit.AfterClass; @@ -86,6 +89,7 @@ public class DriverBaseTest { private static final String dbNamePrefix = "TestDB"; private static final String dbPath = testDir.getAbsolutePath(); private static final String unboundHeapCache = "0"; + // public static String boundHeapCache = "256"; @Parameters(name = "{0}") @@ -294,8 +298,34 @@ public static Iterable data() throws NoSuchMethodException, SecurityEx new boolean[] {false, true, true}, MockDB.class.getDeclaredConstructor(String.class), new Object[] {dbNamePrefix} - } - }); + }, + // Mongo + { + "MongoDB", + new boolean[] { false, false, false }, + MongoDB.class.getDeclaredConstructor(String.class, String.class), + new Object[] { dbNamePrefix + DatabaseTestUtils.getNext(), MongoTestRunner.inst().getConnectionString()} }, + { + "MongoDB+lock", + new boolean[] { true, false, false }, + MongoDB.class.getDeclaredConstructor(String.class, String.class), + new Object[] { dbNamePrefix + DatabaseTestUtils.getNext(), MongoTestRunner.inst().getConnectionString()} }, + { + "MongoDB+heapCache", + new boolean[] { false, true, false }, + MongoDB.class.getDeclaredConstructor(String.class, String.class), + new Object[] { dbNamePrefix + DatabaseTestUtils.getNext(), MongoTestRunner.inst().getConnectionString()} }, + { + "MongoDB+heapCache+lock", + new boolean[] { true, true, false }, + MongoDB.class.getDeclaredConstructor(String.class, String.class), + new Object[] { dbNamePrefix + DatabaseTestUtils.getNext(), MongoTestRunner.inst().getConnectionString()} }, + { + "MongoDB+heapCache+autocommit", + new boolean[] { false, true, true }, + MongoDB.class.getDeclaredConstructor(String.class, String.class), + new Object[] { dbNamePrefix + DatabaseTestUtils.getNext(), MongoTestRunner.inst().getConnectionString()} }, + }); } private IByteArrayKeyValueDatabase db; @@ -332,6 +362,7 @@ public DriverBaseTest( this.args = args; this.dbName = (String) args[0]; this.db = constructor.newInstance(args); + if (props[1]) { this.db = new DatabaseWithCache((AbstractDB) this.db, props[2], "0", false); } @@ -358,7 +389,7 @@ public void open() { assertThat(db.isOpen()).isFalse(); assertThat(db.isClosed()).isTrue(); - if (db.isPersistent()) { + if (db.getPersistenceMethod() == PersistenceMethod.FILE_BASED) { assertThat(db.isCreatedOnDisk()).isFalse(); assertThat(db.getPath().get()).isEqualTo(new File(dbPath, dbName).getAbsolutePath()); } @@ -368,11 +399,14 @@ public void open() { assertThat(db.open()).isTrue(); + // Drop the old db's info if there's any there + db.drop(); + assertThat(db.isOpen()).isTrue(); assertThat(db.isClosed()).isFalse(); assertThat(db.isEmpty()).isTrue(); - if (db.isPersistent()) { + if (db.getPersistenceMethod() == PersistenceMethod.FILE_BASED) { assertThat(db.isCreatedOnDisk()).isTrue(); assertThat(db.getPath().get()).isEqualTo(new File(dbPath, dbName).getAbsolutePath()); } @@ -387,9 +421,13 @@ public void close() { assertThat(db.isOpen()).isTrue(); assertThat(db.isClosed()).isFalse(); - if (db.isPersistent()) { + if (db.getPersistenceMethod() == PersistenceMethod.FILE_BASED) { assertThat(db.isCreatedOnDisk()).isTrue(); assertThat(db.getPath().get()).isEqualTo(new File(dbPath, dbName).getAbsolutePath()); + } else if (db.getPersistenceMethod() == PersistenceMethod.DBMS) { + // Drop the DB before closing the connection for DBMS systems + db.drop(); + assertThat(db.isEmpty()).isTrue(); } assertThat(db.isLocked()).isFalse(); @@ -401,7 +439,7 @@ public void close() { assertThat(db.isClosed()).isTrue(); // for non-persistent DB's, close() should wipe the DB - if (db.isPersistent()) { + if (db.getPersistenceMethod() == PersistenceMethod.FILE_BASED) { assertThat(db.isCreatedOnDisk()).isTrue(); assertThat(FileUtils.deleteRecursively(new File(db.getPath().get()))).isTrue(); assertThat(db.isCreatedOnDisk()).isFalse(); @@ -421,7 +459,7 @@ public void testConcurrentAccess() { public void testOpenSecondInstance() throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { - if (db.isPersistent() && !(db instanceof PersistentMockDB)) { + if (db.getPersistenceMethod() == PersistenceMethod.FILE_BASED && !(db instanceof PersistentMockDB)) { // another connection to same DB should fail on open for all persistent KVDBs IByteArrayKeyValueDatabase otherDatabase = this.constructor.newInstance(this.args); assertThat(otherDatabase.open()).isFalse(); @@ -434,7 +472,7 @@ public void testOpenSecondInstance() @Test public void testPersistence() throws InterruptedException { - if (db.isPersistent()) { + if (db.getPersistenceMethod() != PersistenceMethod.IN_MEMORY) { // adding data // --------------------------------------------------------------------------------------------- assertThat(db.get(k1).isPresent()).isFalse(); @@ -484,7 +522,7 @@ public void testPersistence() throws InterruptedException { @Test public void testBatchPersistence() throws InterruptedException { - if (db.isPersistent()) { + if (db.getPersistenceMethod() != PersistenceMethod.IN_MEMORY) { // adding data // --------------------------------------------------------------------------------------------- assertThat(db.get(k1).isPresent()).isFalse(); @@ -725,7 +763,7 @@ public void testDrop() { @Ignore /** This test is non-deterministic and may fail. If it does, re-run the test suite. */ public void testApproximateDBSize() { - if (db.isPersistent() && !(db instanceof PersistentMockDB)) { + if (db.getPersistenceMethod() == PersistenceMethod.FILE_BASED) { int repeat = 1_000_000; for (int i = 0; i < repeat; i++) { db.put(String.format("%c%09d", 'a' + i % 26, i).getBytes(), "test".getBytes()); @@ -858,7 +896,7 @@ public void testIsEmpty() { /** Checks that data does not persist without explicit commits. */ @Test public void testAutoCommitDisabled() throws InterruptedException { - if (db.isPersistent() && !db.isAutoCommitEnabled()) { + if (db.getPersistenceMethod() != PersistenceMethod.IN_MEMORY && !db.isAutoCommitEnabled()) { // adding data // --------------------------------------------------------------------------------------------- assertThat(db.get(k1).isPresent()).isFalse(); diff --git a/modDbImpl/test/org/aion/db/impl/DriverBenchmarkTest.java b/modDbImpl/test/org/aion/db/impl/DriverBenchmarkTest.java index be8b6a175d..1f5173e696 100644 --- a/modDbImpl/test/org/aion/db/impl/DriverBenchmarkTest.java +++ b/modDbImpl/test/org/aion/db/impl/DriverBenchmarkTest.java @@ -55,6 +55,7 @@ import java.util.Random; import java.util.concurrent.TimeUnit; import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.db.PersistenceMethod; import org.aion.db.impl.h2.H2MVMap; import org.aion.db.impl.leveldb.LevelDB; import org.aion.db.impl.rocksdb.RocksDBConstants; @@ -171,7 +172,7 @@ public void close() { assertTrue(db.isClosed()); // for non-persistant DB's, close() should wipe the DB - if (db.isPersistent()) { + if (db.getPersistenceMethod() == PersistenceMethod.FILE_BASED) { File dbDir = new File(db.getPath().get()); if (dbDir.exists()) { assertTrue(FileUtils.deleteRecursively(dbDir)); diff --git a/modDbImpl/test/org/aion/db/utils/MongoTestRunner.java b/modDbImpl/test/org/aion/db/utils/MongoTestRunner.java new file mode 100644 index 0000000000..72cb2e53ff --- /dev/null +++ b/modDbImpl/test/org/aion/db/utils/MongoTestRunner.java @@ -0,0 +1,149 @@ +package org.aion.db.utils; + +import com.spotify.docker.client.DefaultDockerClient; +import com.spotify.docker.client.DockerClient; +import com.spotify.docker.client.LogStream; +import com.spotify.docker.client.messages.ContainerConfig; +import com.spotify.docker.client.messages.ContainerCreation; +import com.spotify.docker.client.messages.ExecCreation; +import com.spotify.docker.client.messages.HostConfig; +import com.spotify.docker.client.messages.PortBinding; +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.aion.db.impl.DatabaseTestUtils; + + +import static org.junit.Assert.fail; + +/** + * Helper class for spinning up a MongoDB instance to be used for unit tests. + */ +public class MongoTestRunner implements AutoCloseable { + + private int port; + private DockerClient dockerClient; + private String runningDockerContainerId; + private static final String MONGO_IMAGE = "library/mongo:3.6.9"; + + private static class Holder { + static final MongoTestRunner INSTANCE = new MongoTestRunner(); + } + + public static MongoTestRunner inst() { + return Holder.INSTANCE; + } + + private MongoTestRunner() { + try { + // Start by getting a connection to the docker service running on the machine + DefaultDockerClient.Builder clientBuilder = DefaultDockerClient.fromEnv(); + System.out.println("Connecting to docker daemon at " + clientBuilder.uri().toString()); + dockerClient = clientBuilder.build(); + + // Pull the docker image, this will be very quick if it already exists on the machine + dockerClient.pull(MONGO_IMAGE, message -> System.out.println("Docker pull: " + message.status())); + + // Bind container port 27017 to an automatically allocated available host port. + this.port = DatabaseTestUtils.findOpenPort(); + final Map> portBindings = + Map.of("27017", Arrays.asList(PortBinding.of("0.0.0.0", Integer.toString(this.port)))); + final HostConfig hostConfig = HostConfig.builder().portBindings(portBindings).build(); + + // Configure how we want the image to run + ContainerConfig containerConfig = ContainerConfig.builder() + .attachStderr(true) + .hostConfig(hostConfig) + .exposedPorts("27017") + .image(MONGO_IMAGE) + .cmd("--replSet", "rs0", "--noauth", "--nojournal", "--quiet") + .build(); + + // Actually start the container + ContainerCreation creation = dockerClient.createContainer(containerConfig); + dockerClient.startContainer(creation.id()); + this.runningDockerContainerId = creation.id(); + + // Next we run a command to initialize the mongo server's replicas set and admin accounts + String[] initializationCommands = {"mongo", "--eval", "rs.initiate()"}; + tryInitializeDb(initializationCommands, 30, 100); + + // Finally, add a shutdown hook to kill the Mongo server when the process dies + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + close(); + } catch (Exception e) { + e.printStackTrace(); + fail("Failed to close MongoDB connection"); + } + })); + + } catch (Exception e) { + e.printStackTrace(); + fail("Error encountered when initializing mongo docker image. Make sure docker service is running"); + } + } + + /** + * Helper method to run some initialization command on Mongo with some retry logic if the command fails. Since it's + * not determinate how long starting the database will take, we need this retry logic. + * @param initializationCommands The command to actually run + * @param retriesRemaining How many more times to retry the command if it fails + * @param pauseTimeMillis How long to pause between retries + * @throws InterruptedException Thrown when the thread gets interrupted trying to sleep. + */ + private void tryInitializeDb(String[] initializationCommands, int retriesRemaining, long pauseTimeMillis) + throws InterruptedException { + + Exception exception = null; + String execOutput = ""; + try { + final ExecCreation execCreation = dockerClient.execCreate( + this.runningDockerContainerId, initializationCommands, + DockerClient.ExecCreateParam.attachStdout(), + DockerClient.ExecCreateParam.attachStderr(), + DockerClient.ExecCreateParam.detach(false)); + final LogStream output = dockerClient.execStart(execCreation.id()); + execOutput = output.readFully(); + } catch (Exception e) { + exception = e; + } + + // We can't get the exit code, but look for an expected message in the output to determine success + if (exception != null || !execOutput.contains("Using a default configuration for the set")) { + // This is the case that the command didn't work + if (retriesRemaining == 0) { + // We're out of retries, we should fail + if (exception != null) { + exception.printStackTrace(); + } + + fail("Failed to initialize MongoDB, no retries remaining. Output was: " + execOutput); + } else { + Thread.sleep(pauseTimeMillis); + tryInitializeDb(initializationCommands, retriesRemaining - 1, pauseTimeMillis); + } + } + } + + /** + * Returns the connection string to be used to connect to the started Mongo instance + * @return The connection string. + */ + public String getConnectionString() { + return String.format("mongodb://localhost:%d", this.port); + } + + @Override + public void close() throws Exception { + if (this.dockerClient != null && this.runningDockerContainerId != null) { + System.out.println("Killing mongo docker container"); + this.dockerClient.killContainer(this.runningDockerContainerId); + this.dockerClient.removeContainer(this.runningDockerContainerId); + this.dockerClient.close(); + this.dockerClient = null; + this.runningDockerContainerId = null; + } + } +} diff --git a/modDbImpl/test_resources/mongo/start_mongo_local.sh b/modDbImpl/test_resources/mongo/start_mongo_local.sh new file mode 100755 index 0000000000..b681c6d6eb --- /dev/null +++ b/modDbImpl/test_resources/mongo/start_mongo_local.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +# This is a script which can be run to spin up a Mongo DB instance on your local machine using docker + +# First, start up a docker image running Mongo +docker run --name MONGO_LOCAL -d -p 37017:27017 library/mongo:3.6 --replSet rs0 + +# Wait 5 seconds after this finishes for mongo to start itself +sleep 5s + +# Initiatlize the replica set +docker exec -it MONGO_LOCAL mongo --eval 'rs.initiate()' + +echo "Successfully started mongo" +echo "Change config.xml database section with mongodb://localhost:37017 and mongodb" diff --git a/modMcf/src/org/aion/mcf/config/Cfg.java b/modMcf/src/org/aion/mcf/config/Cfg.java index 2e503d3e13..cba2aec30c 100644 --- a/modMcf/src/org/aion/mcf/config/Cfg.java +++ b/modMcf/src/org/aion/mcf/config/Cfg.java @@ -403,7 +403,11 @@ public void setLogDir(File _logDirectory) { } public String getDatabasePath() { - return getDatabaseDir().getAbsolutePath(); + if (getDb().isFileBased()) { + return getDatabaseDir().getAbsolutePath(); + } else { + return getDb().getPath(); + } } public File getDatabaseDir() { diff --git a/modMcf/src/org/aion/mcf/config/CfgDb.java b/modMcf/src/org/aion/mcf/config/CfgDb.java index acc2b729b2..ee8da3e566 100644 --- a/modMcf/src/org/aion/mcf/config/CfgDb.java +++ b/modMcf/src/org/aion/mcf/config/CfgDb.java @@ -35,6 +35,7 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import javax.xml.stream.XMLStreamWriter; +import org.aion.base.db.PersistenceMethod; import org.aion.base.util.Utils; import org.aion.db.impl.DBVendor; @@ -278,6 +279,11 @@ public String toXML() { } } + public boolean isFileBased() { + DBVendor vendor = DBVendor.fromString(this.vendor); + return vendor.isFileBased(); + } + public String getPath() { return path; } @@ -371,7 +377,7 @@ public Map asProperties() { DBVendor vendor = DBVendor.fromString( entry.getValue().asProperties().getProperty(Props.DB_TYPE)); - isPersistent = vendor.getPersistence(); + isPersistent = vendor.isFileBased(); } } @@ -383,7 +389,7 @@ public Map asProperties() { props.setProperty(Props.DB_TYPE, this.vendor); props.setProperty(Props.ENABLE_DB_COMPRESSION, String.valueOf(this.compression)); props.setProperty(Props.CHECK_INTEGRITY, String.valueOf(this.check_integrity)); - boolean isPersistent = DBVendor.fromString(this.vendor).getPersistence(); + boolean isPersistent = DBVendor.fromString(this.vendor).isFileBased(); props.setProperty(Props.PERSISTENT, String.valueOf(isPersistent)); props.setProperty(Props.ENABLE_DB_CACHE, "true"); diff --git a/modMcf/src/org/aion/mcf/db/AbstractRepository.java b/modMcf/src/org/aion/mcf/db/AbstractRepository.java index a20bfdd090..6c9c2da85d 100644 --- a/modMcf/src/org/aion/mcf/db/AbstractRepository.java +++ b/modMcf/src/org/aion/mcf/db/AbstractRepository.java @@ -144,13 +144,12 @@ protected void initializeDatabasesAndCaches() throws InvalidFilePathException { // * TODO: this is hack There should be some information on the // * persistence of the DB so that we do not have to manually check. // * Currently this information exists within - // * {@link DBVendor#getPersistence()}, but is not utilized. + // * {@link DBVendor#getPersistenceMethod()}, but is not utilized. // */ // if (this.cfg.getActiveVendor().equals(DBVendor.MOCKDB.toValue())) { // LOG.warn("WARNING: Active vendor is set to MockDB, data will not persist"); // } else { - // if persistence is required if (Boolean.valueOf(cfg.getDatabaseConfig(Names.DEFAULT).getProperty(Props.PERSISTENT))) { // verify user-provided path File f = new File(this.cfg.getDbPath()); diff --git a/modMcf/src/org/aion/mcf/db/DatabaseUtils.java b/modMcf/src/org/aion/mcf/db/DatabaseUtils.java index 1847207614..8ec8352f24 100644 --- a/modMcf/src/org/aion/mcf/db/DatabaseUtils.java +++ b/modMcf/src/org/aion/mcf/db/DatabaseUtils.java @@ -32,6 +32,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.Properties; import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.db.PersistenceMethod; import org.aion.db.impl.DatabaseFactory; import org.aion.db.impl.DatabaseFactory.Props; import org.aion.mcf.db.exception.InvalidFilePathException; @@ -56,7 +57,7 @@ public static IByteArrayKeyValueDatabase connectAndOpen(Properties info, Logger } // check persistence status - if (!db.isCreatedOnDisk()) { + if (!db.isCreatedOnDisk() && db.getPersistenceMethod() != PersistenceMethod.DBMS) { LOG.error( "Database <{}> cannot be saved to disk for <{}>.", info.getProperty(Props.DB_TYPE), diff --git a/script/prepack.sh b/script/prepack.sh index b5bce569e2..7cca82245a 100755 --- a/script/prepack.sh +++ b/script/prepack.sh @@ -33,7 +33,7 @@ if [ "$noGui" != "true" ] && [ ! -d "$JAVAFX_PATH" ]; then fi module_path=$JDK_PATH/jmods -add_modules="java.base,java.xml,java.logging,java.management,jdk.unsupported,jdk.sctp" +add_modules="java.base,java.xml,java.logging,java.management,jdk.unsupported,jdk.sctp,java.security.sasl" # generate aion runtime if [ "$noGui" != "true" ]; then module_path="$module_path:$JAVAFX_PATH"