Ledger activate unified accounts cache wrapper (#1939)

* Activate `LedgerRef` wrapper for `AccountsCache` details: `accounts_cache.nim` methods are indirectly processed by the wrapper methods from `ledger.nim`. This works for all sources except `test_state_db.nim` where the source `accounts_cache.nim` is included (rather than imported) in order to access objects privy to the very source. * Provide facility to switch to a preselected `LedgerRef` type details: Can be set as suggestion when initialising `CommonRef` * Update `CoreDb` test suite for better time tracking details: + Allow time logging by pre-defined block intervals + Print `CoreDb`/`Ledger`profiling results (if enabled)
status-im · Dec 18, 2023 · 6ba6aa4 · 6ba6aa4
1 parent 4c9617e
commit 6ba6aa4
Show file tree

Hide file tree

Showing 52 changed files with 342 additions and 219 deletions.
diff --git a/hive_integration/nodocker/pyspec/test_env.nim b/hive_integration/nodocker/pyspec/test_env.nim
@@ -18,7 +18,7 @@ import
     config,
     constants,
     transaction,
-    db/accounts_cache,
+    db/ledger,
     core/chain,
     core/tx_pool,
     rpc,

diff --git a/nimbus/common/common.nim b/nimbus/common/common.nim
@@ -14,7 +14,7 @@ import
   chronicles,
   eth/trie/trie_defs,
   ../core/[pow, clique, casper],
-  ../db/[core_db, storage_types],
+  ../db/[core_db, ledger, storage_types],
   ../utils/[utils, ec_recover],
   ".."/[constants, errors],
   "."/[chain_config, evmforks, genesis, hardforks]
@@ -93,6 +93,13 @@ type
     pos: CasperRef
       ## Proof Of Stake descriptor
 
+    ldgType: LedgerType
+      ## Optional suggestion for the ledger cache to be used as state DB
+
+const
+  CommonLedgerTypeDefault* = LegacyAccountsCache
+    ## Default ledger type to use, see `ldgType` above
+
 # ------------------------------------------------------------------------------
 # Forward declarations
 # ------------------------------------------------------------------------------
@@ -138,8 +145,9 @@ proc init(com      : CommonRef,
           networkId: NetworkId,
           config   : ChainConfig,
           genesis  : Genesis,
-          avoidStateDb: bool
+          ldgType  : LedgerType,
             ) {.gcsafe, raises: [CatchableError].} =
+
   config.daoCheck()
 
   com.db          = db
@@ -148,6 +156,7 @@ proc init(com      : CommonRef,
   com.forkTransitionTable = config.toForkTransitionTable()
   com.networkId   = networkId
   com.syncProgress= SyncProgress()
+  com.ldgType     = ldgType
 
   # Initalise the PoA state regardless of whether it is needed on the current
   # network. For non-PoA networks this descriptor is ignored.
@@ -173,7 +182,7 @@ proc init(com      : CommonRef,
       time: some(genesis.timestamp)
     ))
     com.genesisHeader = toGenesisHeader(genesis,
-      com.currentFork, com.db, avoidStateDb)
+      com.currentFork, com.db, com.ldgType)
     com.setForkId(com.genesisHeader)
     com.pos.timestamp = genesis.timestamp
   else:
@@ -208,14 +217,15 @@ proc getTdIfNecessary(com: CommonRef, blockHash: Hash256): Option[DifficultyInt]
 # Public constructors
 # ------------------------------------------------------------------------------
 
-proc new*(_: type CommonRef,
-          db: CoreDbRef,
-          pruneTrie: bool = true,
-          networkId: NetworkId = MainNet,
-          params = networkParams(MainNet),
-          avoidStateDb = false
-            ): CommonRef
-            {.gcsafe, raises: [CatchableError].} =
+proc new*(
+    _: type CommonRef;
+    db: CoreDbRef;
+    pruneTrie: bool = true;
+    networkId: NetworkId = MainNet;
+    params = networkParams(MainNet);
+    ldgType = CommonLedgerTypeDefault;
+      ): CommonRef
+      {.gcsafe, raises: [CatchableError].} =
 
   ## If genesis data is present, the forkIds will be initialized
   ## empty data base also initialized with genesis block
@@ -226,16 +236,17 @@ proc new*(_: type CommonRef,
     networkId,
     params.config,
     params.genesis,
-    avoidStateDb)
-
-proc new*(_: type CommonRef,
-          db: CoreDbRef,
-          config: ChainConfig,
-          pruneTrie: bool = true,
-          networkId: NetworkId = MainNet,
-          avoidStateDb = false
-            ): CommonRef
-            {.gcsafe, raises: [CatchableError].} =
+    ldgType)
+
+proc new*(
+    _: type CommonRef;
+    db: CoreDbRef;
+    config: ChainConfig;
+    pruneTrie: bool = true;
+    networkId: NetworkId = MainNet;
+    ldgType = CommonLedgerTypeDefault;
+      ): CommonRef
+      {.gcsafe, raises: [CatchableError].} =
 
   ## There is no genesis data present
   ## Mainly used for testing without genesis
@@ -246,7 +257,7 @@ proc new*(_: type CommonRef,
     networkId,
     config,
     nil,
-    avoidStateDb)
+    ldgType)
 
 proc clone*(com: CommonRef, db: CoreDbRef): CommonRef =
   ## clone but replace the db
@@ -265,7 +276,8 @@ proc clone*(com: CommonRef, db: CoreDbRef): CommonRef =
     consensusType: com.consensusType,
     pow          : com.pow,
     poa          : com.poa,
-    pos          : com.pos
+    pos          : com.pos,
+    ldgType      : com.ldgType
   )
 
 proc clone*(com: CommonRef): CommonRef =
@@ -495,6 +507,9 @@ func syncHighest*(com: CommonRef): BlockNumber =
 func syncReqRelaxV2*(com: CommonRef): bool =
   com.syncReqRelaxV2
 
+func ledgerType*(com: CommonRef): LedgerType =
+  com.ldgType
+
 # ------------------------------------------------------------------------------
 # Setters
 # ------------------------------------------------------------------------------

diff --git a/nimbus/common/genesis.nim b/nimbus/common/genesis.nim
@@ -14,7 +14,7 @@ import
   std/tables,
   eth/[common, eip1559],
   eth/trie/trie_defs,
-  ../db/[accounts_cache, core_db, distinct_tries, state_db/read_write],
+  ../db/[ledger, core_db, state_db/read_write],
   ../constants,
   ./chain_config
 
@@ -56,6 +56,11 @@ type
     rootHash: GenesisRootHashFn
     getTrie: GenesisGetTrieFn
 
+const
+  GenesisLedgerTypeDefault* = LedgerType(0)
+    ## Default ledger type to use, `LedgerType(0)` uses  `AccountStateDB`
+    ## rather than a `Ledger` variant.
+
 # ------------------------------------------------------------------------------
 # Private functions
 # ------------------------------------------------------------------------------
@@ -93,8 +98,12 @@ proc initStateDbledgerRef(db: CoreDbRef; pruneTrie: bool): GenesisLedgerRef =
       sdb.getTrie())
 
 
-proc initAccountsLedgerRef(db: CoreDbRef; pruneTrie: bool): GenesisLedgerRef =
-  let ac = AccountsCache.init(db, emptyRlpHash, pruneTrie)
+proc initAccountsLedgerRef(
+    db: CoreDbRef;
+    pruneTrie: bool;
+    ledgerType: LedgerType;
+     ): GenesisLedgerRef =
+  let ac = ledgerType.init(db, emptyRlpHash, pruneTrie)
 
   GenesisLedgerRef(
     addAccount: proc(
@@ -114,7 +123,7 @@ proc initAccountsLedgerRef(db: CoreDbRef; pruneTrie: bool): GenesisLedgerRef =
         address: EthAddress;
         slot: UInt256;
         val: UInt256;
-          ) {.rlpRaise.} =
+          ) =
       ac.setStorage(address, slot, val),
 
     commit: proc() =
@@ -124,7 +133,7 @@ proc initAccountsLedgerRef(db: CoreDbRef; pruneTrie: bool): GenesisLedgerRef =
       ac.rootHash(),
 
     getTrie: proc(): CoreDbMptRef =
-      ac.rawTrie.mpt)
+      ac.getMpt())
 
 # ------------------------------------------------------------------------------
 # Public functions
@@ -133,12 +142,12 @@ proc initAccountsLedgerRef(db: CoreDbRef; pruneTrie: bool): GenesisLedgerRef =
 proc newStateDB*(
     db: CoreDbRef;
     pruneTrie: bool;
-    avoidStateDb = false;
+    ledgerType = LedgerType(0);
       ): GenesisLedgerRef =
-  ## The flag `avoidStateDb` is set `false` for compatibility with legacy apps
-  ## `(see `test_state_network`).
-  if avoidStateDb:
-    db.initAccountsLedgerRef pruneTrie
+  ## The flag `ledgerType` is set to zero for compatibility with legacy apps
+  ## (see `test_state_network`).
+  if ledgerType != LedgerType(0):
+    db.initAccountsLedgerRef(pruneTrie, ledgerType)
   else:
     db.initStateDbledgerRef pruneTrie
 
@@ -217,27 +226,27 @@ proc toGenesisHeader*(
     genesis: Genesis;
     fork: HardFork;
     db = CoreDbRef(nil);
-    avoidStateDb = false;
+    ledgerType = GenesisLedgerTypeDefault;
       ): BlockHeader
       {.gcsafe, raises: [CatchableError].} =
   ## Generate the genesis block header from the `genesis` and `config`
   ## argument value.
   let
     db  = if db.isNil: newCoreDbRef LegacyDbMemory else: db
-    sdb = newStateDB(db, pruneTrie = true, avoidStateDb)
+    sdb = newStateDB(db, pruneTrie = true, ledgerType)
   toGenesisHeader(genesis, sdb, fork)
 
 proc toGenesisHeader*(
     params: NetworkParams;
     db = CoreDbRef(nil);
-    avoidStateDb = false;
+    ledgerType = GenesisLedgerTypeDefault;
       ): BlockHeader
       {.raises: [CatchableError].} =
   ## Generate the genesis block header from the `genesis` and `config`
   ## argument value.
   let map  = toForkTransitionTable(params.config)
   let fork = map.toHardFork(forkDeterminationInfo(0.toBlockNumber, params.genesis.timestamp))
-  toGenesisHeader(params.genesis, fork, db, avoidStateDb)
+  toGenesisHeader(params.genesis, fork, db, ledgerType)
 
 # ------------------------------------------------------------------------------
 # End

diff --git a/nimbus/core/chain/persist_blocks.nim b/nimbus/core/chain/persist_blocks.nim
@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018 Status Research & Development GmbH
+# Copyright (c) 2018-2023 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -11,6 +11,7 @@
 {.push raises: [].}
 
 import
+  ../../db/ledger,
   ../../vm_state,
   ../../vm_types,
   ../clique/clique_verify,
@@ -107,8 +108,11 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
     when not defined(release):
       if validationResult == ValidationResult.Error and
          body.transactions.calcTxRoot == header.txRoot:
-        dumpDebuggingMetaData(c.com, header, body, vmState)
-        warn "Validation error. Debugging metadata dumped."
+        if c.com.ledgerType == LegacyAccountsCache:
+          dumpDebuggingMetaData(c.com, header, body, vmState)
+          warn "Validation error. Debugging metadata dumped."
+        else:
+          warn "Validation error", blockNumber=header.blockNumber
 
     if validationResult != ValidationResult.OK:
       return validationResult

diff --git a/nimbus/core/dao.nim b/nimbus/core/dao.nim
@@ -8,7 +8,7 @@
 # at your option. This file may not be copied, modified, or distributed except
 # according to those terms.
 
-import eth/common, stew/byteutils, ../db/accounts_cache
+import eth/common, stew/byteutils, ../db/ledger
 
 const
   # DAOForkBlockExtra is the block header extra-data field to set for the DAO fork
@@ -145,7 +145,7 @@ const
 # ApplyDAOHardFork modifies the state database according to the DAO hard-fork
 # rules, transferring all balances of a set of DAO accounts to a single refund
 # contract.
-proc applyDAOHardFork*(statedb: var AccountsCache) =
+proc applyDAOHardFork*(statedb: LedgerRef) =
   const zero = 0.u256
   # Move every DAO account and extra-balance account funds into the refund contract
   for address in DAODrainList:

diff --git a/nimbus/core/executor/calculate_reward.nim b/nimbus/core/executor/calculate_reward.nim
@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018 Status Research & Development GmbH
+# Copyright (c) 2018-2023 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -9,7 +9,7 @@
 # according to those terms.
 
 import
-  ../../db/accounts_cache,
+  ../../db/ledger,
   ../../common/common,
   ../../vm_state,
   ../../vm_types

diff --git a/nimbus/core/executor/executor_helpers.nim b/nimbus/core/executor/executor_helpers.nim
@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018 Status Research & Development GmbH
+# Copyright (c) 2018-2023 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -12,7 +12,7 @@
 
 import
   ../../common/common,
-  ../../db/accounts_cache,
+  ../../db/ledger,
   ../../vm_state,
   ../../vm_types,
   eth/[bloom]

diff --git a/nimbus/core/executor/process_block.nim b/nimbus/core/executor/process_block.nim
@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018 Status Research & Development GmbH
+# Copyright (c) 2018-2023 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -12,7 +12,7 @@ import
   ../../utils/utils,
   ../../common/common,
   ../../constants,
-  ../../db/accounts_cache,
+  ../../db/ledger,
   ../../transaction,
   ../../vm_state,
   ../../vm_types,

diff --git a/nimbus/core/executor/process_transaction.nim b/nimbus/core/executor/process_transaction.nim
@@ -13,7 +13,7 @@
 import
   std/strutils,
   ../../common/common,
-  ../../db/accounts_cache,
+  ../../db/ledger,
   ../../transaction/call_evm,
   ../../transaction/call_common,
   ../../transaction,
@@ -37,7 +37,7 @@ proc eip1559BaseFee(header: BlockHeader; fork: EVMFork): UInt256 =
     result = header.baseFee
 
 proc commitOrRollbackDependingOnGasUsed(
-    vmState: BaseVMState, accTx: SavePoint,
+    vmState: BaseVMState, accTx: LedgerSpRef,
     header: BlockHeader, tx: Transaction,
     gasBurned: GasInt, priorityFee: GasInt):
     Result[GasInt, string] {.raises: [].} =

diff --git a/nimbus/core/tx_pool/tx_chain.nim b/nimbus/core/tx_pool/tx_chain.nim
@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018 Status Research & Development GmbH
+# Copyright (c) 2022-2023 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -13,10 +13,9 @@
 ##
 
 import
-  std/times,
   ../../common/common,
   ../../constants,
-  ../../db/accounts_cache,
+  ../../db/ledger,
   ../../utils/utils,
   ../../vm_state,
   ../../vm_types,
@@ -156,7 +155,7 @@ proc update(dh: TxChainRef; parent: BlockHeader)
   let
     timestamp = dh.getTimestamp(parent)
     db  = dh.com.db
-    acc = AccountsCache.init(db, parent.stateRoot, dh.com.pruneTrie)
+    acc = dh.com.ledgerType.init(db, parent.stateRoot, dh.com.pruneTrie)
     fee = if dh.com.isLondon(parent.blockNumber + 1, timestamp):
             some(dh.com.baseFeeGet(parent).uint64.u256)
           else:

diff --git a/nimbus/core/tx_pool/tx_tasks/tx_packer.nim b/nimbus/core/tx_pool/tx_tasks/tx_packer.nim
@@ -18,7 +18,7 @@ import
   chronicles,
   eth/[keys, rlp],
   stew/sorted_set,
-  ../../../db/[accounts_cache, core_db],
+  ../../../db/[ledger, core_db],
   ../../../common/common,
   ../../../utils/utils,
   ../../../constants,