Allow EVM state to be flushed to disk every Nth block instead of every block

Makefile

@@ -31,9 +31,9 @@ BINANCE_TGORACLE_DIR=$(GOPATH)/src/$(PKG_BINANCE_TGORACLE)
 #       specific commit.
 GO_LOOM_GIT_REV = HEAD
 # Specifies the loomnetwork/transfer-gateway branch/revision to use.
-TG_GIT_REV = HEAD
+TG_GIT_REV = in-memory-evmstate
 # loomnetwork/go-ethereum loomchain branch
-ETHEREUM_GIT_REV = 1fb6138d017a4309105d91f187c126cf979c93f9
+ETHEREUM_GIT_REV = in-memory-statedb
 # use go-plugin we get 'timeout waiting for connection info' error
 HASHICORP_GIT_REV = f4c3476bd38585f9ec669d10ed1686abd52b9961
 LEVIGO_GIT_REV = c42d9e0ca023e2198120196f842701bb4c55d7b9
@@ -197,14 +197,15 @@ $(BINANCE_TGORACLE_DIR):
 	git clone -q git@github.com:loomnetwork/binance-tgoracle.git $@
 	cd $(BINANCE_TGORACLE_DIR) && git checkout master && git pull && git checkout $(BINANCE_TG_GIT_REV)
 
-validators-tool: $(TRANSFER_GATEWAY_DIR)
-	go build -tags gateway -o e2e/validators-tool $(PKG)/e2e/cmd
-
-deps: $(PLUGIN_DIR) $(GO_ETHEREUM_DIR) $(SSHA3_DIR)
-	# Temp workaround for https://github.com/prometheus/procfs/issues/221
+$(PROMETHEUS_PROCFS_DIR):
 	git clone -q git@github.com:prometheus/procfs $(PROMETHEUS_PROCFS_DIR)
 	cd $(PROMETHEUS_PROCFS_DIR) && git checkout master && git pull && git checkout d3b299e382e6acf1baa852560d862eca4ff643c8
 
+validators-tool: $(TRANSFER_GATEWAY_DIR)
+	go build -tags gateway -o e2e/validators-tool $(PKG)/e2e/cmd
+
+deps: $(PLUGIN_DIR) $(GO_ETHEREUM_DIR) $(SSHA3_DIR) $(PROMETHEUS_PROCFS_DIR)
+
 	go get \
 		golang.org/x/crypto/ed25519 \
 		google.golang.org/grpc \

app.go

@@ -352,6 +352,7 @@ type Application struct {
 	EventHandler
 	ReceiptHandlerProvider
 	EvmAuxStore *evmaux.EvmAuxStore
+	EvmStore    *store.EvmStore
 	blockindex.BlockIndexStore
 	CreateValidatorManager   ValidatorsManagerFactoryFunc
 	CreateChainConfigManager ChainConfigManagerFactoryFunc

app_test.go

@@ -60,7 +60,7 @@ func mockMultiWriterStore(flushInterval int64) (*store.MultiWriterAppStore, erro
 		return nil, err
 	}
 	memDb, _ = db.LoadMemDB()
-	evmStore := store.NewEvmStore(memDb, 100)
+	evmStore := store.NewEvmStore(memDb, 100, 0)
 	multiWriterStore, err := store.NewMultiWriterAppStore(iavlStore, evmStore, false)
 	if err != nil {
 		return nil, err

builtin/plugins/sample_go_contract/sample_go_contract_test.go

@@ -58,8 +58,7 @@ func deployContractToEVM(ctx *plugin.FakeContextWithEVM, filename string, caller
 	}
 	byteCode := common.FromHex(string(hexByteCode))
 	byteCode, err = hex.DecodeString(string(hexByteCode))
-
-	vm := evm.NewLoomVm(ctx.State, nil, nil, nil, false)
+	vm := evm.NewLoomVm(ctx.State, ctx.EvmStore, nil, nil, nil, false)
 	_, contractAddr, err = vm.Create(caller, byteCode, loom.NewBigUIntFromInt(0))
 	if err != nil {
 		return contractAddr, err

cmd/loom/db/db.go

@@ -14,8 +14,6 @@ func NewDBCommand() *cobra.Command {
 	cmd.AddCommand(
 		newPruneDBCommand(),
 		newCompactDBCommand(),
-		newDumpEVMStateCommand(),
-		newDumpEVMStateMultiWriterAppStoreCommand(),
 		newDumpEVMStateFromEvmDB(),
 		newGetEvmHeightCommand(),
 		newGetAppHeightCommand(),

cmd/loom/db/evm.go

@@ -3,7 +3,6 @@
 package db
 
 import (
-	"context"
 	"fmt"
 	"math"
 	"path"
@@ -14,235 +13,18 @@ import (
 	gstate "github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
-	"github.com/loomnetwork/loomchain"
 	"github.com/loomnetwork/loomchain/cmd/loom/common"
 	cdb "github.com/loomnetwork/loomchain/db"
-	"github.com/loomnetwork/loomchain/events"
-	"github.com/loomnetwork/loomchain/evm"
-	"github.com/loomnetwork/loomchain/log"
-	"github.com/loomnetwork/loomchain/plugin"
-	"github.com/loomnetwork/loomchain/receipts"
-	registry "github.com/loomnetwork/loomchain/registry/factory"
 	"github.com/loomnetwork/loomchain/store"
 	"github.com/spf13/cobra"
-	abci "github.com/tendermint/tendermint/abci/types"
-	dbm "github.com/tendermint/tendermint/libs/db"
 )
 
-func newDumpEVMStateCommand() *cobra.Command {
-	var appHeight int64
-
-	cmd := &cobra.Command{
-		Use:   "evm-dump",
-		Short: "Dumps EVM state stored at a specific block height",
-		RunE: func(cmd *cobra.Command, args []string) error {
-			cfg, err := common.ParseConfig()
-			if err != nil {
-				return err
-			}
-
-			db, err := dbm.NewGoLevelDB(cfg.DBName, cfg.RootPath())
-			if err != nil {
-				return err
-			}
-			appStore, err := store.NewIAVLStore(db, 0, appHeight, 0)
-			if err != nil {
-				return err
-			}
-
-			eventHandler := loomchain.NewDefaultEventHandler(events.NewLogEventDispatcher())
-
-			regVer, err := registry.RegistryVersionFromInt(cfg.RegistryVersion)
-			if err != nil {
-				return err
-			}
-			createRegistry, err := registry.NewRegistryFactory(regVer)
-			if err != nil {
-				return err
-			}
-
-			receiptHandlerProvider := receipts.NewReceiptHandlerProvider(
-				eventHandler,
-				cfg.EVMPersistentTxReceiptsMax,
-				nil,
-			)
-
-			// TODO: This should use snapshot obtained from appStore.ReadOnlyState()
-			storeTx := store.WrapAtomic(appStore).BeginTx()
-			state := loomchain.NewStoreState(
-				context.Background(),
-				storeTx,
-				abci.Header{
-					Height: appStore.Version(),
-				},
-				// it is possible to load the block hash from the TM block store, but probably don't
-				// need it for just dumping the EVM state
-				nil,
-				nil,
-			)
-
-			var newABMFactory plugin.NewAccountBalanceManagerFactoryFunc
-			if evm.EVMEnabled && cfg.EVMAccountsEnabled {
-				newABMFactory = plugin.NewAccountBalanceManagerFactory
-			}
-
-			var accountBalanceManager evm.AccountBalanceManager
-			if newABMFactory != nil {
-				pvm := plugin.NewPluginVM(
-					common.NewDefaultContractsLoader(cfg),
-					state,
-					createRegistry(state),
-					eventHandler,
-					log.Default,
-					newABMFactory,
-					receiptHandlerProvider.Writer(),
-					receiptHandlerProvider.Reader(),
-				)
-				createABM, err := newABMFactory(pvm)
-				if err != nil {
-					return err
-				}
-				accountBalanceManager = createABM(true)
-				if err != nil {
-					return err
-				}
-			}
-
-			vm, err := evm.NewLoomEvm(state, accountBalanceManager, nil, false)
-			if err != nil {
-				return err
-			}
-
-			fmt.Printf("\n--- EVM state at app height %d ---\n%s\n", appStore.Version(), string(vm.RawDump()))
-			return nil
-		},
-	}
-
-	cmdFlags := cmd.Flags()
-	cmdFlags.Int64Var(&appHeight, "app-height", 0, "Dump EVM state as it was the specified app height")
-	return cmd
-}
-
-func newDumpEVMStateMultiWriterAppStoreCommand() *cobra.Command {
-	var appHeight int64
-	var evmDBName string
-	cmd := &cobra.Command{
-		Use:   "evm-dump-2",
-		Short: "Dumps EVM state stored at a specific block height from MultiWriterAppStore",
-		RunE: func(cmd *cobra.Command, args []string) error {
-			cfg, err := common.ParseConfig()
-			if err != nil {
-				return err
-			}
-
-			db, err := dbm.NewGoLevelDB(cfg.DBName, cfg.RootPath())
-			if err != nil {
-				return err
-			}
-			evmDB, err := cdb.LoadDB(
-				"goleveldb",
-				evmDBName,
-				cfg.RootPath(),
-				256,
-				4,
-				false,
-			)
-			if err != nil {
-				return err
-			}
-			iavlStore, err := store.NewIAVLStore(db, 0, appHeight, 0)
-			if err != nil {
-				return err
-			}
-			evmStore := store.NewEvmStore(evmDB, 100)
-			if err := evmStore.LoadVersion(iavlStore.Version()); err != nil {
-				return err
-			}
-
-			appStore, err := store.NewMultiWriterAppStore(iavlStore, evmStore, false)
-			if err != nil {
-				return err
-			}
-			eventHandler := loomchain.NewDefaultEventHandler(events.NewLogEventDispatcher())
-
-			regVer, err := registry.RegistryVersionFromInt(cfg.RegistryVersion)
-			if err != nil {
-				return err
-			}
-			createRegistry, err := registry.NewRegistryFactory(regVer)
-			if err != nil {
-				return err
-			}
-
-			receiptHandlerProvider := receipts.NewReceiptHandlerProvider(
-				eventHandler,
-				cfg.EVMPersistentTxReceiptsMax,
-				nil,
-			)
-
-			// TODO: This should use snapshot obtained from appStore.ReadOnlyState()
-			storeTx := store.WrapAtomic(appStore).BeginTx()
-			state := loomchain.NewStoreState(
-				context.Background(),
-				storeTx,
-				abci.Header{
-					Height: appStore.Version(),
-				},
-				// it is possible to load the block hash from the TM block store, but probably don't
-				// need it for just dumping the EVM state
-				nil,
-				nil,
-			)
-
-			var newABMFactory plugin.NewAccountBalanceManagerFactoryFunc
-			if evm.EVMEnabled && cfg.EVMAccountsEnabled {
-				newABMFactory = plugin.NewAccountBalanceManagerFactory
-			}
-
-			var accountBalanceManager evm.AccountBalanceManager
-			if newABMFactory != nil {
-				pvm := plugin.NewPluginVM(
-					common.NewDefaultContractsLoader(cfg),
-					state,
-					createRegistry(state),
-					eventHandler,
-					log.Default,
-					newABMFactory,
-					receiptHandlerProvider.Writer(),
-					receiptHandlerProvider.Reader(),
-				)
-				createABM, err := newABMFactory(pvm)
-				if err != nil {
-					return err
-				}
-				accountBalanceManager = createABM(true)
-				if err != nil {
-					return err
-				}
-			}
-
-			vm, err := evm.NewLoomEvm(state, accountBalanceManager, nil, false)
-			if err != nil {
-				return err
-			}
-
-			fmt.Printf("\n--- EVM state at app height %d ---\n%s\n", appStore.Version(), string(vm.RawDump()))
-			return nil
-		},
-	}
-
-	cmdFlags := cmd.Flags()
-	cmdFlags.Int64Var(&appHeight, "app-height", 0, "Dump EVM state as it was the specified app height")
-	cmdFlags.StringVar(&evmDBName, "evmdb-name", "evm", "Name of EVM state database")
-	return cmd
-}
-
 func newDumpEVMStateFromEvmDB() *cobra.Command {
 	var appHeight int64
 	var evmDBName string
 	var dumpStorageTrie bool
 	cmd := &cobra.Command{
-		Use:   "evm-dump-3",
+		Use:   "evm-dump",
 		Short: "Dumps EVM state stored at a specific block height from evm.db",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cfg, err := common.ParseConfig()
@@ -262,7 +44,7 @@ func newDumpEVMStateFromEvmDB() *cobra.Command {
 				return err
 			}
 
-			evmStore := store.NewEvmStore(evmDB, 100)
+			evmStore := store.NewEvmStore(evmDB, 100, cfg.AppStore.IAVLFlushInterval)
 			if err := evmStore.LoadVersion(appHeight); err != nil {
 				return err
 			}
@@ -271,21 +53,7 @@ func newDumpEVMStateFromEvmDB() *cobra.Command {
 
 			fmt.Printf("version: %d, root: %x\n", version, root)
 
-			// TODO: This should use snapshot obtained from appStore.ReadOnlyState()
-			storeTx := store.WrapAtomic(evmStore).BeginTx()
-			state := loomchain.NewStoreState(
-				context.Background(),
-				storeTx,
-				abci.Header{
-					Height: appHeight,
-				},
-				// it is possible to load the block hash from the TM block store, but probably don't
-				// need it for just dumping the EVM state
-				nil,
-				nil,
-			)
-
-			srcStateDB := gstate.NewDatabase(evm.NewLoomEthdb(state, nil))
+			srcStateDB := gstate.NewDatabase(store.NewLoomEthDB(evmStore, nil))
 			srcStateDBTrie, err := srcStateDB.OpenTrie(evmRoot)
 			if err != nil {
 				fmt.Printf("cannot open trie, %s\n", evmRoot.Hex())
@@ -364,7 +132,7 @@ func newGetEvmHeightCommand() *cobra.Command {
 			}
 			defer db.Close()
 
-			evmStore := store.NewEvmStore(db, 100)
+			evmStore := store.NewEvmStore(db, 100, 0)
 			if err := evmStore.LoadVersion(math.MaxInt64); err != nil {
 				return err
 			}