diff --git a/CHANGES.md b/CHANGES.md index 0c3730c38d..188b63a141 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -60,6 +60,18 @@ To be released. ### Backward-incompatible storage format changes + - (Libplanet.RocksDBStore) The blocks and transactions became stored in + multiple databases. Each block and transaction belongs to a partition + of the database, according to its epoch unit, which is its Unix timestamp. + Every epoch is divided by certain seconds, configured by `RocksDBStore()` + constructor's `txEpochUnitSeconds` and `blockEpochUnitSeconds` parameters + (86400 by default). [[#1183], [#1194]] + - (Libplanet.RocksDBStore) Continue on partitioning of database, + `RocksDBStore()` is manage database connection by LRU Cache. + The max size of connection cache is configured by `RocksDBStore()` + constructor's `dbConnectionCacheSize` parameters (100 + by default). [[#1183], [#1194]] + ### Added APIs - Added `Block.CurrentProtocolVersion` constant. [[#1142], [#1147]] @@ -244,10 +256,11 @@ To be released. [#1180]: https://github.com/planetarium/libplanet/pull/1180 [#1181]: https://github.com/planetarium/libplanet/pull/1181 [#1182]: https://github.com/planetarium/libplanet/pull/1182 +[#1183]: https://github.com/planetarium/libplanet/issues/1183 [#1184]: https://github.com/planetarium/libplanet/pull/1184 [#1185]: https://github.com/planetarium/libplanet/pull/1185 [#1186]: https://github.com/planetarium/libplanet/pull/1186 - +[#1194]: https://github.com/planetarium/libplanet/pull/1194 Version 0.10.3 -------------- diff --git a/Libplanet.RocksDBStore/RocksDBStore.cs b/Libplanet.RocksDBStore/RocksDBStore.cs index 20e34c5360..46e1b53f5f 100644 --- a/Libplanet.RocksDBStore/RocksDBStore.cs +++ b/Libplanet.RocksDBStore/RocksDBStore.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Security.Cryptography; +using System.Threading; using Libplanet.Blocks; using Libplanet.Store; using Libplanet.Tx; @@ -21,9 +22,11 @@ namespace Libplanet.RocksDBStore /// public class RocksDBStore : BaseStore { - private const string BlockDbName = "block"; + private const string BlockDbRootPathName = "block"; + private const string BlockIndexDbName = "blockindex"; private const string BlockPerceptionDbName = "blockpercept"; - private const string TxDbName = "tx"; + private const string TxDbRootPathName = "tx"; + private const string TxIndexDbName = "txindex"; private const string StagedTxDbName = "stagedtx"; private const string ChainDbName = "chain"; @@ -44,13 +47,20 @@ public class RocksDBStore : BaseStore private readonly DbOptions _options; private readonly string _path; + private readonly int _txEpochUnitSeconds; + private readonly int _blockEpochUnitSeconds; - private readonly RocksDb _blockDb; + private readonly RocksDb _blockIndexDb; private readonly RocksDb _blockPerceptionDb; - private readonly RocksDb _txDb; + private readonly LruCache _blockDbCache; + private readonly RocksDb _txIndexDb; + private readonly LruCache _txDbCache; private readonly RocksDb _stagedTxDb; private readonly RocksDb _chainDb; + private readonly ReaderWriterLockSlim _rwTxLock; + private readonly ReaderWriterLockSlim _rwBlockLock; + /// /// Creates a new . /// @@ -65,13 +75,22 @@ public class RocksDBStore : BaseStore /// option. /// The number to configure max_log_file_size /// RocksDB option. + /// The number of Transaction to store in DB + /// for each epoch. 86400 by default. + /// The number of block to store in DB + /// for each epoch. 86400 by default. + /// The capacity of the block and transaction + /// RocksDB connection cache. 100 by default. public RocksDBStore( string path, int blockCacheSize = 512, int txCacheSize = 1024, ulong? maxTotalWalSize = null, ulong? keepLogFileNum = null, - ulong? maxLogFileSize = null + ulong? maxLogFileSize = null, + int txEpochUnitSeconds = 86400, + int blockEpochUnitSeconds = 86400, + int dbConnectionCacheSize = 100 ) { _logger = Log.ForContext(); @@ -92,6 +111,16 @@ public RocksDBStore( _blockCache = new LruCache, BlockDigest>(capacity: blockCacheSize); _path = path; + _txEpochUnitSeconds = txEpochUnitSeconds > 0 + ? txEpochUnitSeconds + : throw new ArgumentException( + "It must be greater than 0.", + nameof(txEpochUnitSeconds)); + _blockEpochUnitSeconds = blockEpochUnitSeconds > 0 + ? blockEpochUnitSeconds + : throw new ArgumentException( + "It must be greater than 0.", + nameof(blockEpochUnitSeconds)); _options = new DbOptions() .SetCreateIfMissing(); @@ -110,10 +139,10 @@ public RocksDBStore( _options = _options.SetMaxLogFileSize(maxLogFileSizeValue); } - _blockDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(BlockDbName)); + _blockIndexDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(BlockIndexDbName)); _blockPerceptionDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(BlockPerceptionDbName)); - _txDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(TxDbName)); + _txIndexDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(TxIndexDbName)); _stagedTxDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StagedTxDbName)); // When opening a DB in a read-write mode, you need to specify all Column Families that @@ -121,6 +150,22 @@ public RocksDBStore( var chainDbColumnFamilies = GetColumnFamilies(_options, ChainDbName); _chainDb = RocksDBUtils.OpenRocksDb( _options, RocksDbPath(ChainDbName), chainDbColumnFamilies); + + _rwTxLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); + _rwBlockLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); + + _blockDbCache = new LruCache(dbConnectionCacheSize); + _blockDbCache.SetPreRemoveDataMethod(db => + { + db.Dispose(); + return true; + }); + _txDbCache = new LruCache(dbConnectionCacheSize); + _txDbCache.SetPreRemoveDataMethod(db => + { + db.Dispose(); + return true; + }); } /// @@ -319,7 +364,7 @@ public override IEnumerable IterateTransactionIds() { byte[] prefix = TxKeyPrefix; - foreach (Iterator it in IterateDb(_txDb, prefix) ) + foreach (Iterator it in IterateDb(_txIndexDb, prefix) ) { byte[] key = it.Key(); byte[] txIdBytes = key.Skip(prefix.Length).ToArray(); @@ -338,16 +383,31 @@ public override Transaction GetTransaction(TxId txid) } byte[] key = TxKey(txid); - byte[] bytes = _txDb.Get(key); - - if (bytes is null) + if (!(_txIndexDb.Get(key) is byte[] txDbNameBytes)) { return null; } - Transaction tx = Transaction.Deserialize(bytes, false); - _txCache.AddOrUpdate(txid, tx); - return tx; + string txDbName = RocksDBStoreBitConverter.GetString(txDbNameBytes); + _rwTxLock.EnterReadLock(); + try + { + if (!_txDbCache.TryGetValue(txDbName, out RocksDb txDb)) + { + txDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(txDbName)); + _txDbCache.AddOrUpdate(txDbName, txDb); + } + + byte[] txBytes = txDb.Get(key); + + Transaction tx = Transaction.Deserialize(txBytes, false); + _txCache.AddOrUpdate(txid, tx); + return tx; + } + finally + { + _rwTxLock.ExitReadLock(); + } } /// @@ -359,14 +419,30 @@ public override void PutTransaction(Transaction tx) } byte[] key = TxKey(tx.Id); - - if (!(_txDb.Get(key) is null)) + if (!(_txIndexDb.Get(key) is null)) { return; } - _txDb.Put(key, tx.Serialize(true)); - _txCache.AddOrUpdate(tx.Id, tx); + long timestamp = tx.Timestamp.ToUnixTimeSeconds(); + string txDbName = $"epoch{(int)timestamp / _txEpochUnitSeconds}"; + _rwTxLock.EnterWriteLock(); + try + { + if (!_txDbCache.TryGetValue(txDbName, out RocksDb txDb)) + { + txDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(txDbName)); + _txDbCache.AddOrUpdate(txDbName, txDb); + } + + txDb.Put(key, tx.Serialize(true)); + _txIndexDb.Put(key, RocksDBStoreBitConverter.GetBytes(txDbName)); + _txCache.AddOrUpdate(tx.Id, tx); + } + finally + { + _rwTxLock.ExitWriteLock(); + } } /// @@ -374,15 +450,31 @@ public override bool DeleteTransaction(TxId txid) { byte[] key = TxKey(txid); - if (_txDb.Get(key) is null) + if (!(_txIndexDb.Get(key) is byte[] txDbNameBytes)) { return false; } - _txCache.Remove(txid); - _txDb.Remove(key); + _rwTxLock.EnterWriteLock(); + try + { + string txDbName = RocksDBStoreBitConverter.GetString(txDbNameBytes); + if (!_txDbCache.TryGetValue(txDbName, out RocksDb txDb)) + { + txDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(txDbName)); + _txDbCache.AddOrUpdate(txDbName, txDb); + } + + _txCache.Remove(txid); + _txIndexDb.Remove(key); + txDb.Remove(key); - return true; + return true; + } + finally + { + _rwTxLock.ExitWriteLock(); + } } /// @@ -395,7 +487,7 @@ public override bool ContainsTransaction(TxId txId) byte[] key = TxKey(txId); - return !(_txDb.Get(key) is null); + return !(_txIndexDb.Get(key) is null); } /// @@ -403,7 +495,7 @@ public override IEnumerable> IterateBlockHashes() { byte[] prefix = BlockKeyPrefix; - foreach (Iterator it in IterateDb(_blockDb, prefix)) + foreach (Iterator it in IterateDb(_blockIndexDb, prefix)) { byte[] key = it.Key(); byte[] hashBytes = key.Skip(prefix.Length).ToArray(); @@ -422,17 +514,32 @@ public override IEnumerable> IterateBlockHashes() } byte[] key = BlockKey(blockHash); - byte[] bytes = _blockDb.Get(key); - - if (bytes is null) + if (!(_blockIndexDb.Get(key) is byte[] blockDbNameBytes)) { return null; } - BlockDigest blockDigest = BlockDigest.Deserialize(bytes); + _rwBlockLock.EnterReadLock(); + try + { + string blockDbName = RocksDBStoreBitConverter.GetString(blockDbNameBytes); + if (!_blockDbCache.TryGetValue(blockDbName, out RocksDb blockDb)) + { + blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName)); + _blockDbCache.AddOrUpdate(blockDbName, blockDb); + } + + byte[] blockBytes = blockDb.Get(key); - _blockCache.AddOrUpdate(blockHash, blockDigest); - return blockDigest; + BlockDigest blockDigest = BlockDigest.Deserialize(blockBytes); + + _blockCache.AddOrUpdate(blockHash, blockDigest); + return blockDigest; + } + finally + { + _rwBlockLock.ExitReadLock(); + } } /// @@ -445,19 +552,37 @@ public override void PutBlock(Block block) byte[] key = BlockKey(block.Hash); - if (!(_blockDb.Get(key) is null)) + if (!(_blockIndexDb.Get(key) is null)) { return; } + long timestamp = block.Timestamp.ToUnixTimeSeconds(); + foreach (Transaction tx in block.Transactions) { PutTransaction(tx); } - byte[] value = block.ToBlockDigest().Serialize(); - _blockDb.Put(key, value); - _blockCache.AddOrUpdate(block.Hash, block.ToBlockDigest()); + _rwBlockLock.EnterWriteLock(); + try + { + string blockDbName = $"epoch{timestamp / _blockEpochUnitSeconds}"; + if (!_blockDbCache.TryGetValue(blockDbName, out RocksDb blockDb)) + { + blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName)); + _blockDbCache.AddOrUpdate(blockDbName, blockDb); + } + + byte[] value = block.ToBlockDigest().Serialize(); + blockDb.Put(key, value); + _blockIndexDb.Put(key, RocksDBStoreBitConverter.GetBytes(blockDbName)); + _blockCache.AddOrUpdate(block.Hash, block.ToBlockDigest()); + } + finally + { + _rwBlockLock.ExitWriteLock(); + } } /// @@ -465,15 +590,30 @@ public override bool DeleteBlock(HashDigest blockHash) { byte[] key = BlockKey(blockHash); - if (_blockDb.Get(key) is null) + if (!(_blockIndexDb.Get(key) is byte[] blockDbNameByte)) { return false; } - _blockCache.Remove(blockHash); - _blockDb.Remove(key); + _rwBlockLock.EnterWriteLock(); + try + { + string blockDbName = RocksDBStoreBitConverter.GetString(blockDbNameByte); + if (!_blockDbCache.TryGetValue(blockDbName, out RocksDb blockDb)) + { + blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName)); + _blockDbCache.AddOrUpdate(blockDbName, blockDb); + } - return true; + _blockCache.Remove(blockHash); + _blockIndexDb.Remove(key); + blockDb.Remove(key); + return true; + } + finally + { + _rwBlockLock.ExitWriteLock(); + } } /// @@ -486,7 +626,7 @@ public override bool ContainsBlock(HashDigest blockHash) byte[] key = BlockKey(blockHash); - return !(_blockDb.Get(key) is null); + return !(_blockIndexDb.Get(key) is null); } /// @@ -570,10 +710,23 @@ public override long CountBlocks() public override void Dispose() { _chainDb?.Dispose(); - _txDb?.Dispose(); - _blockDb?.Dispose(); + _txIndexDb?.Dispose(); + _blockIndexDb?.Dispose(); _blockPerceptionDb?.Dispose(); _stagedTxDb?.Dispose(); + foreach (var db in _txDbCache.Values) + { + db.Dispose(); + } + + _txDbCache.Clear(); + + foreach (var db in _blockDbCache.Values) + { + db.Dispose(); + } + + _blockDbCache.Clear(); } private byte[] BlockKey(HashDigest blockHash) @@ -653,6 +806,12 @@ private ColumnFamilies GetColumnFamilies(DbOptions options, string dbName) return columnFamilies; } + private string TxDbPath(string dbName) => + Path.Combine(RocksDbPath(TxDbRootPathName), dbName); + + private string BlockDbPath(string dbName) => + Path.Combine(RocksDbPath(BlockDbRootPathName), dbName); + private string RocksDbPath(string dbName) => Path.Combine(_path, dbName); } } diff --git a/Libplanet.RocksDBStore/RocksDBUtils.cs b/Libplanet.RocksDBStore/RocksDBUtils.cs index 82629c1e54..98ee881160 100644 --- a/Libplanet.RocksDBStore/RocksDBUtils.cs +++ b/Libplanet.RocksDBStore/RocksDBUtils.cs @@ -1,4 +1,5 @@ #nullable enable +using System.IO; using RocksDbSharp; namespace Libplanet.RocksDBStore @@ -8,6 +9,11 @@ internal static class RocksDBUtils internal static RocksDb OpenRocksDb( DbOptions options, string dbPath, ColumnFamilies? columnFamilies = null) { + if (!Directory.Exists(dbPath)) + { + Directory.CreateDirectory(dbPath); + } + return columnFamilies is null ? RocksDb.Open(options, dbPath) : RocksDb.Open(options, dbPath, columnFamilies); diff --git a/Libplanet/Libplanet.csproj b/Libplanet/Libplanet.csproj index a53c7eacac..2f20140122 100644 --- a/Libplanet/Libplanet.csproj +++ b/Libplanet/Libplanet.csproj @@ -68,7 +68,7 @@ https://docs.libplanet.io/ all - + all