Skip to content

Commit

Permalink
Allow EVM state to be flushed to disk every Nth block instead of ever…
Browse files Browse the repository at this point in the history 
…y block (#1532)

These changes should reduce the amount of data written to evm.db by only flushing EVM state to disk periodically instead of every block, similar to how data is written to app.db.

* Bump default number of cached EVM state roots to 500

The flush interval is currently set to 100, so the cache must be a bit
larger than that, otherwise in-mem-only roots are likely to be evicted
too quickly and EVMState.GetSnapshot() will end up returning older state
than it should.

* Eliminate PruningIAVLStore

We haven't used this store in ages and have no intention of doing so
again because it has been replaced by the IAVL flushing interval.
Finally getting rid of this store now because the snapshot code for the
remaining stores needs to be updated and I don't want to waste time
modifying code that doesn't get used.

* Verify state roots in EVMState.GetSnapshot()

Due to the flush interval in the EvmStore some EVM state roots may never
be written to disk, so it's important to check that the root of the
state returned by EVMState.GetSnapshot() at a particular height matches
the EVM state root stored in the app store at that height, otherwise the
EVM state and app state returned by Application.ReadOnlyState() may end
up being out of sync.

* Hack VersionedCachingStore to get the e2e tests passing

The EVM root in the VersionedCachingStore cache wasn't being updated in
VersionedCachingStore.SaveVersion() which meant that the
snapshots obtained from the VersionedCachingStore contained a stale EVM
root that didn't match the EVM state in the snapshot.

This hack updates the cached EVM root in
VersionedCachingStore.SaveVersion() but it's not a proper fix for the
issue, the VersionedCachingStore shouldn't be aware of the EVM root at
all. The most sensible thing to do is probably to eliminate the
MultiWriterAppStore itself and update Application.Commit() to call
EvmStore.Commit(), and then store the EVM root in the
VersionedCachingStore.

* Add EvmStore.FlushInterval setting to loom.yml

This setting works similarly to AppStore.IAVLFlushInterval.

Co-authored-by: Vadim Macagon <vadim.macagon@gmail.com>
  • Loading branch information
@pathornteng @enlight
pathornteng and enlight committed Jan 28, 2020
1 parent 1bb40ac commit a6fba1c
Show file tree
Hide file tree
Showing 36 changed files with 864 additions and 1,294 deletions.
12 changes: 7 additions & 5 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ GO_LOOM_GIT_REV = HEAD
# Specifies the loomnetwork/transfer-gateway branch/revision to use.
TG_GIT_REV = HEAD
# loomnetwork/go-ethereum loomchain branch
ETHEREUM_GIT_REV = 6128fa1a8c767035d3da6ef0c27ebb7778ce3713
ETHEREUM_GIT_REV = cce1b3f69354033160583e5576169f9b309ee62e
# use go-plugin we get 'timeout waiting for connection info' error
HASHICORP_GIT_REV = f4c3476bd38585f9ec669d10ed1686abd52b9961
LEVIGO_GIT_REV = c42d9e0ca023e2198120196f842701bb4c55d7b9
Expand Down Expand Up @@ -197,13 +197,15 @@ $(BINANCE_TGORACLE_DIR):
git clone -q git@github.com:loomnetwork/binance-tgoracle.git $@
cd $(BINANCE_TGORACLE_DIR) && git checkout master && git pull && git checkout $(BINANCE_TG_GIT_REV)

validators-tool: $(TRANSFER_GATEWAY_DIR)
go build -tags gateway -o e2e/validators-tool $(PKG)/e2e/cmd

deps: $(PLUGIN_DIR) $(GO_ETHEREUM_DIR) $(SSHA3_DIR)
$(PROMETHEUS_PROCFS_DIR):
# Temp workaround for https://github.com/prometheus/procfs/issues/221
git clone -q git@github.com:prometheus/procfs $(PROMETHEUS_PROCFS_DIR)
cd $(PROMETHEUS_PROCFS_DIR) && git checkout master && git pull && git checkout d3b299e382e6acf1baa852560d862eca4ff643c8

validators-tool: $(TRANSFER_GATEWAY_DIR)
go build -tags gateway -o e2e/validators-tool $(PKG)/e2e/cmd

deps: $(PLUGIN_DIR) $(GO_ETHEREUM_DIR) $(SSHA3_DIR) $(PROMETHEUS_PROCFS_DIR)
# Lock down Prometheus golang client to v1.2.1 (newer versions use a different protobuf version)
git clone -q git@github.com:prometheus/client_golang $(GOPATH)/src/github.com/prometheus/client_golang
cd $(GOPATH)/src/github.com/prometheus/client_golang && git checkout master && git pull && git checkout v1.2.1
Expand Down
70 changes: 56 additions & 14 deletions app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ import (
"encoding/binary"
"encoding/hex"
"fmt"
"sync/atomic"
"time"
"unsafe"

"github.com/loomnetwork/go-loom/config"
"github.com/loomnetwork/go-loom/util"
Expand Down Expand Up @@ -51,6 +53,7 @@ type State interface {
SetFeature(string, bool)
SetMinBuildNumber(uint64)
ChangeConfigSetting(name, value string) error
EVMState() *EVMState
}

type StoreState struct {
Expand All @@ -60,6 +63,7 @@ type StoreState struct {
validators loom.ValidatorSet
getValidatorSet GetValidatorSet
config *cctypes.Config
evmState *EVMState
}

var _ = State(&StoreState{})
Expand Down Expand Up @@ -101,6 +105,11 @@ func (s *StoreState) WithOnChainConfig(config *cctypes.Config) *StoreState {
return s
}

func (s *StoreState) WithEVMState(evmState *EVMState) *StoreState {
s.evmState = evmState
return s
}

func (s *StoreState) Range(prefix []byte) plugin.RangeData {
return s.store.Range(prefix)
}
Expand Down Expand Up @@ -141,6 +150,10 @@ func (s *StoreState) Context() context.Context {
return s.ctx
}

func (s *StoreState) EVMState() *EVMState {
return s.evmState
}

const (
featurePrefix = "feature"
MinBuildKey = "minbuild"
Expand Down Expand Up @@ -234,6 +247,7 @@ func (s *StoreState) WithContext(ctx context.Context) State {
ctx: ctx,
validators: s.validators,
getValidatorSet: s.getValidatorSet,
evmState: s.evmState,
}
}

Expand All @@ -244,6 +258,7 @@ func (s *StoreState) WithPrefix(prefix []byte) State {
ctx: s.ctx,
validators: s.validators,
getValidatorSet: s.getValidatorSet,
evmState: s.evmState,
}
}

Expand Down Expand Up @@ -347,7 +362,7 @@ type CommittedTx struct {
}

type Application struct {
lastBlockHeader abci.Header
lastBlockHeader unsafe.Pointer // *abci.Header
curBlockHeader abci.Header
curBlockHash []byte
Store store.VersionedKVStore
Expand All @@ -368,6 +383,7 @@ type Application struct {
config *cctypes.Config
childTxRefs []evmaux.ChildTxRef // links Tendermint txs to EVM txs
ReceiptsVersion int32
EVMState *EVMState
committedTxs []CommittedTx
}

Expand Down Expand Up @@ -512,7 +528,7 @@ func (a *Application) BeginBlock(req abci.RequestBeginBlock) abci.ResponseBeginB
a.curBlockHeader,
a.curBlockHash,
a.GetValidatorSet,
).WithOnChainConfig(a.config)
).WithOnChainConfig(a.config).WithEVMState(a.EVMState)
contractUpkeepHandler, err := a.CreateContractUpkeepHandler(upkeepState)
if err != nil {
panic(err)
Expand All @@ -532,7 +548,7 @@ func (a *Application) BeginBlock(req abci.RequestBeginBlock) abci.ResponseBeginB
a.curBlockHeader,
nil,
a.GetValidatorSet,
).WithOnChainConfig(a.config)
).WithOnChainConfig(a.config).WithEVMState(a.EVMState)

validatorManager, err := a.CreateValidatorManager(state)
if err != registry.ErrNotFound {
Expand Down Expand Up @@ -600,7 +616,7 @@ func (a *Application) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock {
a.curBlockHeader,
nil,
a.GetValidatorSet,
).WithOnChainConfig(a.config)
).WithOnChainConfig(a.config).WithEVMState(a.EVMState)

validatorManager, err := a.CreateValidatorManager(state)
if err != registry.ErrNotFound {
Expand Down Expand Up @@ -657,6 +673,10 @@ func (a *Application) CheckTx(txBytes []byte) abci.ResponseCheckTx {
a.GetValidatorSet,
).WithOnChainConfig(a.config)

if a.EVMState != nil {
state = state.WithEVMState(a.EVMState.Clone())
}

// Receipts & events generated in CheckTx must be discarded since the app state changes they
// reflect aren't persisted.
defer a.ReceiptHandlerProvider.Store().DiscardCurrentReceipt()
Expand Down Expand Up @@ -692,7 +712,7 @@ func (a *Application) DeliverTx(txBytes []byte) abci.ResponseDeliverTx {
a.curBlockHeader,
a.curBlockHash,
a.GetValidatorSet,
).WithOnChainConfig(a.config)
).WithOnChainConfig(a.config).WithEVMState(a.EVMState)

var r abci.ResponseDeliverTx

Expand Down Expand Up @@ -725,7 +745,7 @@ func (a *Application) processTx(storeTx store.KVStoreTx, txBytes []byte, isCheck
a.curBlockHeader,
a.curBlockHash,
a.GetValidatorSet,
).WithOnChainConfig(a.config)
).WithOnChainConfig(a.config).WithEVMState(a.EVMState)

receiptHandler := a.ReceiptHandlerProvider.Store()
defer receiptHandler.DiscardCurrentReceipt()
Expand Down Expand Up @@ -778,7 +798,7 @@ func (a *Application) deliverTx2(storeTx store.KVStoreTx, txBytes []byte) abci.R
a.curBlockHeader,
a.curBlockHash,
a.GetValidatorSet,
).WithOnChainConfig(a.config)
).WithOnChainConfig(a.config).WithEVMState(a.EVMState)

receiptHandler := a.ReceiptHandlerProvider.Store()
defer receiptHandler.DiscardCurrentReceipt()
Expand Down Expand Up @@ -830,13 +850,19 @@ func (a *Application) Commit() abci.ResponseCommit {
commitBlockLatency.With(lvs...).Observe(time.Since(begin).Seconds())
}(time.Now())

if a.EVMState != nil {
// Commit EVM state changes to the EvmStore
if err := a.EVMState.Commit(); err != nil {
panic(err)
}
}

appHash, _, err := a.Store.SaveVersion()
if err != nil {
panic(err)
}

height := a.curBlockHeader.GetHeight()

if err := a.EvmAuxStore.SaveChildTxRefs(a.childTxRefs); err != nil {
// TODO: consider panic instead
log.Error("Failed to save Tendermint -> EVM tx hash refs", "height", height, "err", err)
Expand All @@ -851,7 +877,8 @@ func (a *Application) Commit() abci.ResponseCommit {

// Update the last block header before emitting events in case the subscribers attempt to access
// the latest committed state as soon as they receive an event.
a.lastBlockHeader = a.curBlockHeader
curBlockHeader := a.curBlockHeader
atomic.StorePointer(&a.lastBlockHeader, unsafe.Pointer(&curBlockHeader))

go func(height int64, blockHeader abci.Header, committedTxs []CommittedTx) {
if err := a.EventHandler.EmitBlockTx(uint64(height), blockHeader.Time); err != nil {
Expand Down Expand Up @@ -904,13 +931,28 @@ func (a *Application) height() int64 {
}

func (a *Application) ReadOnlyState() State {
// TODO: the store snapshot should be created atomically, otherwise the block header might
// not match the state... need to figure out why this hasn't spectacularly failed already
lastBlockHeader := (*abci.Header)(atomic.LoadPointer(&a.lastBlockHeader))
appStateSnapshot, err := a.Store.GetSnapshotAt(lastBlockHeader.Height)
if err != nil {
panic(err)
}

var evmStateSnapshot *EVMState
if a.EVMState != nil {
evmStateSnapshot, err = a.EVMState.GetSnapshot(
lastBlockHeader.Height,
store.GetEVMRootFromAppStore(appStateSnapshot),
)
if err != nil {
panic(err)
}
}

return NewStoreStateSnapshot(
nil,
a.Store.GetSnapshot(),
a.lastBlockHeader,
appStateSnapshot,
*lastBlockHeader,
nil, // TODO: last block hash!
a.GetValidatorSet,
)
).WithEVMState(evmStateSnapshot)
}
2 changes: 1 addition & 1 deletion app_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func mockMultiWriterStore(flushInterval int64) (*store.MultiWriterAppStore, erro
return nil, err
}
memDb, _ = db.LoadMemDB()
evmStore := store.NewEvmStore(memDb, 100)
evmStore := store.NewEvmStore(memDb, 100, 0)
multiWriterStore, err := store.NewMultiWriterAppStore(iavlStore, evmStore, false)
if err != nil {
return nil, err
Expand Down
2 changes: 0 additions & 2 deletions cmd/loom/db/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ func NewDBCommand() *cobra.Command {
cmd.AddCommand(
newPruneDBCommand(),
newCompactDBCommand(),
newDumpEVMStateCommand(),
newDumpEVMStateMultiWriterAppStoreCommand(),
newDumpEVMStateFromEvmDB(),
newGetEvmHeightCommand(),
newGetAppHeightCommand(),
Expand Down
Loading

0 comments on commit a6fba1c

Please sign in to comment.