diff --git a/CHANGELOG.md b/CHANGELOG.md index e2f45b63a55a..c9cc00252bd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,14 @@ to now accept a `codec.JSONMarshaler` for modular serialization of genesis state * (keys) [\#5820](https://github.com/cosmos/cosmos-sdk/pull/5820/) Removed method CloseDB from Keybase interface. * (baseapp) [\#5837](https://github.com/cosmos/cosmos-sdk/issues/5837) Transaction simulation now returns a `SimulationResponse` which contains the `GasInfo` and `Result` from the execution. +* (store) [\#5803](https://github.com/cosmos/cosmos-sdk/pull/5803) The `store.CommitMultiStore` interface now includes the new `snapshots.Snapshotter` interface as well. +* (crypto/keyring) [\#5866](https://github.com/cosmos/cosmos-sdk/pull/5866) Move `Keyring` and `Keybase` implementations and their associated types from `crypto/keys/` to `crypto/keyring/`. +* (crypto) [\#5880](https://github.com/cosmos/cosmos-sdk/pull/5880) Merge `crypto/keys/mintkey` into `crypto`. +* (crypto/keyring) [\#5858](https://github.com/cosmos/cosmos-sdk/pull/5858) Make Keyring store keys by name and address's hexbytes representation. +* (crypto/keyring) [\#5889](https://github.com/cosmos/cosmos-sdk/pull/5889) Deprecate old keybase implementation: + - Remove `Update` from the `Keybase` interface. + - `NewKeyring()` now accepts a new backend: `MemoryBackend`. + - `New()` has been renamed to`NewLegacy()`, which now returns a `LegacyKeybase` type that only allows migration of keys from the legacy keybase to a new keyring. * (client/input) [\#5904](https://github.com/cosmos/cosmos-sdk/pull/5904) Removal of unnecessary `GetCheckPassword`, `PrintPrefixed` functions. * (client/keys) [\#5889](https://github.com/cosmos/cosmos-sdk/pull/5889) Rename `NewKeyBaseFromDir()` -> `NewLegacyKeyBaseFromDir()`. * (crypto) [\#5880](https://github.com/cosmos/cosmos-sdk/pull/5880) Merge `crypto/keys/mintkey` into `crypto`. @@ -99,6 +107,8 @@ information on how to implement the new `Keyring` interface. ### Features * (x/ibc) [\#5588](https://github.com/cosmos/cosmos-sdk/pull/5588) Add [ICS 024 - Host State Machine Requirements](https://github.com/cosmos/ics/tree/master/spec/ics-024-host-requirements) subpackage to `x/ibc` module. +* (baseapp) [\#5803](https://github.com/cosmos/cosmos-sdk/pull/5803) Added support for taking state snapshots at regular height intervals, via options `snapshot-interval` and `snapshot-retention`. +* (store) [\#5803](https://github.com/cosmos/cosmos-sdk/pull/5803) Added `rootmulti.Store` methods for taking and restoring snapshots, based on `iavl.Store` export/import. * (x/ibc) [\#5277](https://github.com/cosmos/cosmos-sdk/pull/5277) `x/ibc` changes from IBC alpha. For more details check the the [`x/ibc/spec`](https://github.com/cosmos/tree/master/x/ibc/spec) directory: * [ICS 002 - Client Semantics](https://github.com/cosmos/ics/tree/master/spec/ics-002-client-semantics) subpackage * [ICS 003 - Connection Semantics](https://github.com/cosmos/ics/blob/master/spec/ics-003-connection-semantics) subpackage diff --git a/baseapp/abci.go b/baseapp/abci.go index 864c7bfb72c9..45887c5e9615 100644 --- a/baseapp/abci.go +++ b/baseapp/abci.go @@ -1,6 +1,7 @@ package baseapp import ( + "errors" "fmt" "os" "sort" @@ -10,6 +11,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/cosmos/cosmos-sdk/codec" + "github.com/cosmos/cosmos-sdk/snapshots" sdk "github.com/cosmos/cosmos-sdk/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" ) @@ -263,6 +265,10 @@ func (app *BaseApp) Commit() (res abci.ResponseCommit) { app.halt() } + if app.snapshotInterval > 0 && uint64(header.Height)%app.snapshotInterval == 0 { + go app.snapshot(header.Height) + } + return abci.ResponseCommit{ Data: commitID.Hash, } @@ -290,6 +296,27 @@ func (app *BaseApp) halt() { os.Exit(0) } +// snapshot takes a snapshot of the current state and prunes any old snapshots. +func (app *BaseApp) snapshot(height int64) { + app.logger.Info("Taking state snapshot", "height", height) + snapshot, err := app.snapshotManager.Take(uint64(height)) + if err != nil { + app.logger.Error("Failed to take state snapshot", "height", height, "err", err) + return + } + app.logger.Info("Completed state snapshot", "height", height, "format", snapshot.Format) + + if snapshotRetention > 0 { + app.logger.Debug("Pruning state snapshots") + pruned, err := app.snapshotManager.Prune(snapshotRetention) + if err != nil { + app.logger.Error("Failed to prune state snapshots", "err", err.Error()) + return + } + app.logger.Debug("Pruned state snapshots", "pruned", pruned) + } +} + // Query implements the ABCI interface. It delegates to CommitMultiStore if it // implements Queryable. func (app *BaseApp) Query(req abci.RequestQuery) abci.ResponseQuery { @@ -316,6 +343,100 @@ func (app *BaseApp) Query(req abci.RequestQuery) abci.ResponseQuery { return sdkerrors.QueryResult(sdkerrors.Wrap(sdkerrors.ErrUnknownRequest, "unknown query path")) } +// ListSnapshots implements the ABCI interface. It delegates to app.snapshotManager if set. +func (app *BaseApp) ListSnapshots(req abci.RequestListSnapshots) abci.ResponseListSnapshots { + resp := abci.ResponseListSnapshots{Snapshots: []*abci.Snapshot{}} + if app.snapshotManager == nil { + return resp + } + + snapshots, err := app.snapshotManager.List() + if err != nil { + app.logger.Error("Failed to list snapshots", "err", err.Error()) + return resp + } + for _, snapshot := range snapshots { + abciSnapshot, err := snapshot.ToABCI() + if err != nil { + app.logger.Error("Failed to list snapshots", "err", err.Error()) + return resp + } + resp.Snapshots = append(resp.Snapshots, &abciSnapshot) + } + + return resp +} + +// LoadSnapshotChunk implements the ABCI interface. It delegates to app.snapshotManager if set. +func (app *BaseApp) LoadSnapshotChunk(req abci.RequestLoadSnapshotChunk) abci.ResponseLoadSnapshotChunk { + if app.snapshotManager == nil { + return abci.ResponseLoadSnapshotChunk{} + } + chunk, err := app.snapshotManager.LoadChunk(req.Height, req.Format, req.Chunk) + if err != nil { + app.logger.Error("Failed to load snapshot chunk", "height", req.Height, "format", req.Format, + "chunk", req.Chunk, "err", err.Error()) + return abci.ResponseLoadSnapshotChunk{} + } + return abci.ResponseLoadSnapshotChunk{Chunk: chunk} +} + +// OfferSnapshot implements the ABCI interface. It delegates to app.snapshotManager if set. +func (app *BaseApp) OfferSnapshot(req abci.RequestOfferSnapshot) abci.ResponseOfferSnapshot { + if req.Snapshot == nil { + app.logger.Error("Received nil snapshot") + return abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT} + } + + snapshot, err := snapshots.SnapshotFromABCI(req.Snapshot) + if err != nil { + app.logger.Error("Failed to decode snapshot metadata", "err", err) + return abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT} + } + err = app.snapshotManager.Restore(snapshot) + switch { + case err == nil: + return abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT} + + case errors.Is(err, snapshots.ErrUnknownFormat): + return abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT_FORMAT} + + case errors.Is(err, snapshots.ErrInvalidMetadata): + app.logger.Error("Rejecting invalid snapshot", "height", req.Snapshot.Height, + "format", req.Snapshot.Format, "err", err.Error()) + return abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT} + + default: + app.logger.Error("Failed to restore snapshot", "height", req.Snapshot.Height, + "format", req.Snapshot.Format, "err", err.Error()) + // We currently don't support resetting the IAVL stores and retrying a different snapshot, + // so we ask Tendermint to abort all snapshot restoration. + return abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ABORT} + } +} + +// ApplySnapshotChunk implements the ABCI interface. It delegates to app.snapshotManager if set. +func (app *BaseApp) ApplySnapshotChunk(req abci.RequestApplySnapshotChunk) abci.ResponseApplySnapshotChunk { + _, err := app.snapshotManager.RestoreChunk(req.Chunk) + switch { + case err == nil: + return abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT} + + case errors.Is(err, snapshots.ErrChunkHashMismatch): + app.logger.Error("Chunk checksum mismatch, rejecting sender and requesting refetch", + "chunk", req.Index, "sender", req.Sender, "err", err) + return abci.ResponseApplySnapshotChunk{ + Result: abci.ResponseApplySnapshotChunk_RETRY, + RefetchChunks: []uint32{req.Index}, + RejectSenders: []string{req.Sender}, + } + + default: + app.logger.Error("Failed to restore snapshot", "err", err.Error()) + return abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ABORT} + } +} + func handleQueryApp(app *BaseApp, path []string, req abci.RequestQuery) abci.ResponseQuery { if len(path) >= 2 { switch path[1] { diff --git a/baseapp/baseapp.go b/baseapp/baseapp.go index 712605a65475..88841bd73e92 100644 --- a/baseapp/baseapp.go +++ b/baseapp/baseapp.go @@ -11,12 +11,14 @@ import ( "github.com/tendermint/tendermint/libs/log" dbm "github.com/tendermint/tm-db" + "github.com/cosmos/cosmos-sdk/snapshots" "github.com/cosmos/cosmos-sdk/store" sdk "github.com/cosmos/cosmos-sdk/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" ) const ( + snapshotRetention = 2 runTxModeCheck runTxMode = iota // Check a transaction runTxModeReCheck // Recheck a (pending) transaction after a commit runTxModeSimulate // Simulate a transaction @@ -62,6 +64,10 @@ type BaseApp struct { // nolint: maligned idPeerFilter sdk.PeerFilter // filter peers by node ID fauxMerkleMode bool // if true, IAVL MountStores uses MountStoresDB for simulation speed. + // manages snapshots, i.e. dumps of app state at certain intervals + snapshotManager *snapshots.Manager + snapshotInterval uint64 // block interval between snapshots - must be multiple of IAVL SnapshotEvery + // volatile states: // // checkState is set on InitChain and reset on Commit diff --git a/baseapp/baseapp_test.go b/baseapp/baseapp_test.go index b65d12f1a7f6..a6ae37a2126d 100644 --- a/baseapp/baseapp_test.go +++ b/baseapp/baseapp_test.go @@ -5,10 +5,12 @@ import ( "encoding/binary" "encoding/json" "fmt" + "math/rand" "os" "strings" "sync" "testing" + "time" "github.com/gogo/protobuf/jsonpb" "github.com/stretchr/testify/assert" @@ -18,6 +20,7 @@ import ( dbm "github.com/tendermint/tm-db" "github.com/cosmos/cosmos-sdk/codec" + "github.com/cosmos/cosmos-sdk/snapshots" "github.com/cosmos/cosmos-sdk/store/rootmulti" store "github.com/cosmos/cosmos-sdk/store/types" sdk "github.com/cosmos/cosmos-sdk/types" @@ -86,6 +89,7 @@ func registerTestCodec(cdc *codec.Codec) { cdc.RegisterConcrete(&txTest{}, "cosmos-sdk/baseapp/txTest", nil) cdc.RegisterConcrete(&msgCounter{}, "cosmos-sdk/baseapp/msgCounter", nil) cdc.RegisterConcrete(&msgCounter2{}, "cosmos-sdk/baseapp/msgCounter2", nil) + cdc.RegisterConcrete(&msgKeyValue{}, "cosmos-sdk/baseapp/msgKeyValue", nil) cdc.RegisterConcrete(&msgNoRoute{}, "cosmos-sdk/baseapp/msgNoRoute", nil) } @@ -103,6 +107,65 @@ func setupBaseApp(t *testing.T, options ...func(*BaseApp)) *BaseApp { return app } +// simple one store baseapp with data and snapshots. Each tx is 1 MB in size (uncompressed). +func setupBaseAppWithSnapshots(t *testing.T, blocks uint, blockTxs int, options ...func(*BaseApp)) (*BaseApp, func()) { + codec := codec.New() + registerTestCodec(codec) + routerOpt := func(bapp *BaseApp) { + bapp.Router().AddRoute(routeMsgKeyValue, func(ctx sdk.Context, msg sdk.Msg) (*sdk.Result, error) { + kv := msg.(*msgKeyValue) + bapp.cms.GetCommitKVStore(capKey2).Set(kv.Key, kv.Value) + return &sdk.Result{}, nil + }) + } + + snapshotDir := os.TempDir() + snapshotStore, err := snapshots.NewStore(dbm.NewMemDB(), snapshotDir) + require.NoError(t, err) + teardown := func() { + os.RemoveAll(snapshotDir) + } + + app := setupBaseApp(t, append(options, + SetSnapshotStore(snapshotStore), + SetPruning(sdk.PruningOptions{ + KeepEvery: 1, + SnapshotEvery: 2, + }), + routerOpt)...) + + app.InitChain(abci.RequestInitChain{}) + + r := rand.New(rand.NewSource(3920758213583)) + keyCounter := 0 + for height := int64(1); height <= int64(blocks); height++ { + app.BeginBlock(abci.RequestBeginBlock{Header: abci.Header{Height: height}}) + for txNum := 0; txNum < blockTxs; txNum++ { + tx := txTest{Msgs: []sdk.Msg{}} + for msgNum := 0; msgNum < 100; msgNum++ { + key := []byte(fmt.Sprintf("%v", keyCounter)) + value := make([]byte, 10000) + _, err := r.Read(value) + require.NoError(t, err) + tx.Msgs = append(tx.Msgs, msgKeyValue{Key: key, Value: value}) + keyCounter++ + } + txBytes, err := codec.MarshalBinaryBare(tx) + require.NoError(t, err) + resp := app.DeliverTx(abci.RequestDeliverTx{Tx: txBytes}) + require.True(t, resp.IsOK(), "%v", resp.String()) + } + app.EndBlock(abci.RequestEndBlock{Height: height}) + app.Commit() + + // Wait for snapshot to be taken, since it happens asynchronously. This + // heuristic is likely to be flaky on low-IO machines. + time.Sleep(time.Duration(int(height)*blockTxs) * 200 * time.Millisecond) + } + + return app, teardown +} + func TestMountStores(t *testing.T) { app := setupBaseApp(t) @@ -585,6 +648,7 @@ func (tx txTest) ValidateBasic() error { return nil } const ( routeMsgCounter = "msgCounter" routeMsgCounter2 = "msgCounter2" + routeMsgKeyValue = "msgKeyValue" ) // ValidateBasic() fails on negative counters. @@ -646,6 +710,26 @@ func (msg msgCounter2) ValidateBasic() error { return sdkerrors.Wrap(sdkerrors.ErrInvalidSequence, "counter should be a non-negative integer") } +// A msg that sets a key/value pair. +type msgKeyValue struct { + Key []byte + Value []byte +} + +func (msg msgKeyValue) Route() string { return routeMsgKeyValue } +func (msg msgKeyValue) Type() string { return "keyValue" } +func (msg msgKeyValue) GetSignBytes() []byte { return nil } +func (msg msgKeyValue) GetSigners() []sdk.AccAddress { return nil } +func (msg msgKeyValue) ValidateBasic() error { + if msg.Key == nil { + return sdkerrors.Wrap(sdkerrors.ErrInvalidRequest, "key cannot be nil") + } + if msg.Value == nil { + return sdkerrors.Wrap(sdkerrors.ErrInvalidRequest, "value cannot be nil") + } + return nil +} + // amino decode func testTxDecoder(cdc *codec.Codec) sdk.TxDecoder { return func(txBytes []byte) (sdk.Tx, error) { @@ -1465,6 +1549,166 @@ func TestQuery(t *testing.T) { require.Equal(t, value, res.Value) } +func TestListSnapshots(t *testing.T) { + app, teardown := setupBaseAppWithSnapshots(t, 5, 4) + defer teardown() + + resp := app.ListSnapshots(abci.RequestListSnapshots{}) + for _, s := range resp.Snapshots { + assert.NotEmpty(t, s.Hash) + assert.NotEmpty(t, s.Metadata) + s.Hash = nil + s.Metadata = nil + } + assert.Equal(t, abci.ResponseListSnapshots{Snapshots: []*abci.Snapshot{ + {Height: 4, Format: 1, Chunks: 2}, + {Height: 2, Format: 1, Chunks: 1}, + }}, resp) +} + +func TestLoadSnapshotChunk(t *testing.T) { + app, teardown := setupBaseAppWithSnapshots(t, 2, 5) + defer teardown() + + testcases := map[string]struct { + height uint64 + format uint32 + chunk uint32 + expectEmpty bool + }{ + "Existing snapshot": {2, 1, 1, false}, + "Missing height": {100, 1, 1, true}, + "Missing format": {2, 2, 1, true}, + "Missing chunk": {2, 1, 9, true}, + "Zero height": {0, 1, 1, true}, + "Zero format": {2, 0, 1, true}, + "Zero chunk": {2, 1, 0, false}, + } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + resp := app.LoadSnapshotChunk(abci.RequestLoadSnapshotChunk{ + Height: tc.height, + Format: tc.format, + Chunk: tc.chunk, + }) + if tc.expectEmpty { + assert.Equal(t, abci.ResponseLoadSnapshotChunk{}, resp) + return + } + assert.NotEmpty(t, resp.Chunk) + }) + } +} + +func TestOfferSnapshot_Errors(t *testing.T) { + // Set up app before test cases, since it's fairly expensive. + app, teardown := setupBaseAppWithSnapshots(t, 0, 0) + defer teardown() + + m := snapshots.Metadata{ChunkHashes: [][]byte{{1}, {2}, {3}}} + metadata, err := m.Marshal() + require.NoError(t, err) + hash := []byte{1, 2, 3} + + testcases := map[string]struct { + snapshot *abci.Snapshot + result abci.ResponseOfferSnapshot_Result + }{ + "nil snapshot": {nil, abci.ResponseOfferSnapshot_REJECT}, + "invalid format": {&abci.Snapshot{ + Height: 1, Format: 9, Chunks: 3, Hash: hash, Metadata: metadata, + }, abci.ResponseOfferSnapshot_REJECT_FORMAT}, + "incorrect chunk count": {&abci.Snapshot{ + Height: 1, Format: 1, Chunks: 2, Hash: hash, Metadata: metadata, + }, abci.ResponseOfferSnapshot_REJECT}, + "no chunks": {&abci.Snapshot{ + Height: 1, Format: 1, Chunks: 0, Hash: hash, Metadata: metadata, + }, abci.ResponseOfferSnapshot_REJECT}, + "invalid metadata serialization": {&abci.Snapshot{ + Height: 1, Format: 1, Chunks: 0, Hash: hash, Metadata: []byte{3, 1, 4}, + }, abci.ResponseOfferSnapshot_REJECT}, + } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + resp := app.OfferSnapshot(abci.RequestOfferSnapshot{Snapshot: tc.snapshot}) + assert.Equal(t, tc.result, resp.Result) + }) + } + + // Offering a snapshot after one has been accepted should error + resp := app.OfferSnapshot(abci.RequestOfferSnapshot{Snapshot: &abci.Snapshot{ + Height: 1, + Format: snapshots.CurrentFormat, + Chunks: 3, + Hash: []byte{1, 2, 3}, + Metadata: metadata, + }}) + require.Equal(t, abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT}, resp) + + resp = app.OfferSnapshot(abci.RequestOfferSnapshot{Snapshot: &abci.Snapshot{ + Height: 2, + Format: snapshots.CurrentFormat, + Chunks: 3, + Hash: []byte{1, 2, 3}, + Metadata: metadata, + }}) + require.Equal(t, abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ABORT}, resp) +} + +func TestApplySnapshotChunk(t *testing.T) { + source, teardown := setupBaseAppWithSnapshots(t, 4, 10) + defer teardown() + + target, teardown := setupBaseAppWithSnapshots(t, 0, 0) + defer teardown() + + // Fetch latest snapshot to restore + respList := source.ListSnapshots(abci.RequestListSnapshots{}) + require.NotEmpty(t, respList.Snapshots) + snapshot := respList.Snapshots[0] + + // Make sure the snapshot has at least 3 chunks + require.GreaterOrEqual(t, snapshot.Chunks, uint32(3), "Not enough snapshot chunks") + + // Begin a snapshot restoration in the target + respOffer := target.OfferSnapshot(abci.RequestOfferSnapshot{Snapshot: snapshot}) + require.Equal(t, abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT}, respOffer) + + // We should be able to pass an invalid chunk and get a verify failure, before reapplying it. + respApply := target.ApplySnapshotChunk(abci.RequestApplySnapshotChunk{ + Index: 0, + Chunk: []byte{9}, + Sender: "sender", + }) + require.Equal(t, abci.ResponseApplySnapshotChunk{ + Result: abci.ResponseApplySnapshotChunk_RETRY, + RefetchChunks: []uint32{0}, + RejectSenders: []string{"sender"}, + }, respApply) + + // Fetch each chunk from the source and apply it to the target + for index := uint32(0); index < snapshot.Chunks; index++ { + respChunk := source.LoadSnapshotChunk(abci.RequestLoadSnapshotChunk{ + Height: snapshot.Height, + Format: snapshot.Format, + Chunk: index, + }) + require.NotNil(t, respChunk.Chunk) + respApply := target.ApplySnapshotChunk(abci.RequestApplySnapshotChunk{ + Index: index, + Chunk: respChunk.Chunk, + }) + require.Equal(t, abci.ResponseApplySnapshotChunk{ + Result: abci.ResponseApplySnapshotChunk_ACCEPT, + }, respApply) + } + + // The target should now have the same hash as the source + assert.Equal(t, source.LastCommitID(), target.LastCommitID()) +} + // Test p2p filter queries func TestP2PQuery(t *testing.T) { addrPeerFilterOpt := func(bapp *BaseApp) { diff --git a/baseapp/options.go b/baseapp/options.go index e3c3629a1351..88ac45563214 100644 --- a/baseapp/options.go +++ b/baseapp/options.go @@ -6,6 +6,7 @@ import ( dbm "github.com/tendermint/tm-db" + "github.com/cosmos/cosmos-sdk/snapshots" "github.com/cosmos/cosmos-sdk/store" sdk "github.com/cosmos/cosmos-sdk/types" ) @@ -13,9 +14,17 @@ import ( // File for storing in-package BaseApp optional functions, // for options that need access to non-exported fields of the BaseApp -// SetPruning sets a pruning option on the multistore associated with the app +// SetPruning sets a pruning option on the multistore associated with the app, and also +// sets the state sync snapshot interval (unless SnapshotEvery is 1, which disables it). func SetPruning(opts sdk.PruningOptions) func(*BaseApp) { - return func(bap *BaseApp) { bap.cms.SetPruning(opts) } + return func(bap *BaseApp) { + if opts.SnapshotEvery > 1 { + bap.snapshotInterval = uint64(opts.SnapshotEvery) + } else { + bap.snapshotInterval = 0 + } + bap.cms.SetPruning(opts) + } } // SetMinGasPrices returns an option that sets the minimum gas prices on the app. @@ -44,6 +53,11 @@ func SetInterBlockCache(cache sdk.MultiStorePersistentCache) func(*BaseApp) { return func(app *BaseApp) { app.setInterBlockCache(cache) } } +// SetSnapshotDB sets the snapshot store. +func SetSnapshotStore(snapshotStore *snapshots.Store) func(*BaseApp) { + return func(app *BaseApp) { app.SetSnapshotStore(snapshotStore) } +} + func (app *BaseApp) SetName(name string) { if app.sealed { panic("SetName() on sealed BaseApp") @@ -151,3 +165,11 @@ func (app *BaseApp) SetRouter(router sdk.Router) { } app.router = router } + +// SetSnapshotStore sets the snapshot store. +func (app *BaseApp) SetSnapshotStore(snapshotStore *snapshots.Store) { + if app.sealed { + panic("SetSnapshotStore() on sealed BaseApp") + } + app.snapshotManager = snapshots.NewManager(snapshotStore, app.cms) +} diff --git a/go.mod b/go.mod index 049600965ca0..049727ad93b5 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/gibson042/canonicaljson-go v1.0.3 github.com/gogo/protobuf v1.3.1 github.com/golang/mock v1.4.3 - github.com/golang/protobuf v1.4.0 + github.com/golang/protobuf v1.4.1 github.com/gorilla/handlers v1.4.2 github.com/gorilla/mux v1.7.4 github.com/hashicorp/golang-lru v0.5.4 @@ -36,4 +36,7 @@ require ( replace github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.2-alpha.regen.1 +// FIXME Temporary until erik/state-sync branch is merged +replace github.com/tendermint/tendermint => ../tendermint + go 1.14 diff --git a/go.sum b/go.sum index af1d79b8091a..8be85884cb4d 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,6 @@ github.com/99designs/keyring v1.1.5 h1:wLv7QyzYpFIyMSwOADq1CLTF9KbjbBfcnfmOGJ64a github.com/99designs/keyring v1.1.5/go.mod h1:7hsVvt2qXgtadGevGJ4ujg+u8m6SpJ5TpHqTozIPqf0= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/ChainSafe/go-schnorrkel v0.0.0-20200102211924-4bcbc698314f h1:4O1om+UVU+Hfcihr1timk8YNXHxzZWgCo7ofnrZRApw= -github.com/ChainSafe/go-schnorrkel v0.0.0-20200102211924-4bcbc698314f/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d h1:nalkkPQcITbvhmL4+C4cKA87NW0tfm3Kl9VXRoPywFg= github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -38,12 +36,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d h1:xG8Pj6Y6J760xwETNmMzmlt38QSwz0BLp1cZ09g27uw= github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a h1:RQMUrEILyYJEoAT34XS/kLu40vC0+po/UfxrBBA4qZE= github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts= @@ -102,8 +98,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= @@ -146,13 +140,14 @@ github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -180,8 +175,6 @@ github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -191,7 +184,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= -github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f h1:8N8XWLZelZNibkhM1FuF+3Ad3YIbgirjdMiVA0eUkaM= github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= github.com/gtank/merlin v0.1.1 h1:eQ90iG7K9pOhtereWsmyRJ6RAwcP4tHTDBHXNg+u5is= github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= @@ -343,10 +335,10 @@ github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= -github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= +github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -367,10 +359,11 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -403,8 +396,6 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.6 h1:breEStsVwemnKh2/s6gMvSdMEkwW0sK8vGStnlVBMCs= -github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -415,8 +406,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= -github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs= github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -446,11 +435,6 @@ github.com/tendermint/go-amino v0.15.1 h1:D2uk35eT4iTsvJd9jWIetzthE5C0/k2QmMFkCN github.com/tendermint/go-amino v0.15.1/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME= github.com/tendermint/iavl v0.13.3 h1:expgBDY1MX+6/3sqrIxGChbTNf9N9aTJ67SH4bPchCs= github.com/tendermint/iavl v0.13.3/go.mod h1:2lE7GiWdSvc7kvT78ncIKmkOjCnp6JEnSb2O7B9htLw= -github.com/tendermint/tendermint v0.33.2 h1:NzvRMTuXJxqSsFed2J7uHmMU5N1CVzSpfi3nCc882KY= -github.com/tendermint/tendermint v0.33.2/go.mod h1:25DqB7YvV1tN3tHsjWoc2vFtlwICfrub9XO6UBO+4xk= -github.com/tendermint/tendermint v0.33.4 h1:NM3G9618yC5PaaxGrcAySc5ylc1PAANeIx42u2Re/jo= -github.com/tendermint/tendermint v0.33.4/go.mod h1:6NW9DVkvsvqmCY6wbRsOo66qGDhMXglRL70aXajvBEA= -github.com/tendermint/tm-db v0.4.1/go.mod h1:JsJ6qzYkCGiGwm5GHl/H5GLI9XLb6qZX7PRe425dHAY= github.com/tendermint/tm-db v0.5.1 h1:H9HDq8UEA7Eeg13kdYckkgwwkQLBnJGgX4PgLJRhieY= github.com/tendermint/tm-db v0.5.1/go.mod h1:g92zWjHpCYlEvQXvy9M168Su8V1IBEeawpXVVBaK4f4= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -484,7 +468,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200406173513-056763e48d71 h1:DOmugCavvUtnUD114C1Wh+UgTgQZ4pMLzXxi1pSt+/Y= @@ -515,7 +498,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -543,12 +525,14 @@ golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -597,17 +581,19 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k= -google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.0 h1:2pJjwYOdkZ9HlN4sWRYBg9ttH5bCOlsueaM+b/oYjwo= +google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0 h1:cJv5/xdbk1NnMPR1VP9+HU6gupuG9MLBoH1r6RHZ2MY= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/server/config/config.go b/server/config/config.go index cba6917deab0..20a9d5870549 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -32,6 +32,12 @@ type BaseConfig struct { // Note: Commitment of state will be attempted on the corresponding block. HaltTime uint64 `mapstructure:"halt-time"` + // SnapshotInterval sets the interval between state snapshots (in blocks). 0 to disable. + SnapshotInterval uint64 `mapstructure:"snapshot-interval"` + + // SnapshotRetention sets the number of recent snapshots to keep. 0 keeps all snapshots. + SnapshotRetention uint32 `mapstructure:"snapshot-retention"` + // InterBlockCache enables inter-block caching. InterBlockCache bool `mapstructure:"inter-block-cache"` @@ -76,11 +82,13 @@ func (c *Config) GetMinGasPrices() sdk.DecCoins { func DefaultConfig() *Config { return &Config{ BaseConfig{ - MinGasPrices: defaultMinGasPrices, InterBlockCache: true, Pruning: store.PruningStrategySyncable, PruningKeepEvery: "0", PruningSnapshotEvery: "0", + MinGasPrices: defaultMinGasPrices, + SnapshotInterval: 0, + SnapshotRetention: 3, }, } } diff --git a/server/config/toml.go b/server/config/toml.go index 744595580c15..97212d6acc29 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -44,6 +44,12 @@ pruning = "{{ .BaseConfig.Pruning }}" # These are applied if and only if the pruning strategy is custom. pruning-keep-every = "{{ .BaseConfig.PruningKeepEvery }}" pruning-snapshot-every = "{{ .BaseConfig.PruningSnapshotEvery }}" + +# State snapshots can be taken at regular height intervals, given by snapshot-interval (0 to +# disable). Old snapshots can be removed by setting snapshot-retention, giving the number +# of recent snapshots to keep (0 to keep all). +snapshot-interval = {{ .BaseConfig.SnapshotInterval }} +snapshot-retention = {{ .BaseConfig.SnapshotRetention }} ` var configTemplate *template.Template diff --git a/server/mock/store.go b/server/mock/store.go index bf8e66a4dc07..ea27c6382073 100644 --- a/server/mock/store.go +++ b/server/mock/store.go @@ -99,6 +99,14 @@ func (ms multiStore) SetInterBlockCache(_ sdk.MultiStorePersistentCache) { panic("not implemented") } +func (ms multiStore) Snapshot(height uint64, format uint32) (<-chan io.ReadCloser, error) { + panic("not implemented") +} + +func (ms multiStore) Restore(height uint64, format uint32, chunks <-chan io.ReadCloser) error { + panic("not implemented") +} + var _ sdk.KVStore = kvStore{} type kvStore struct { diff --git a/server/start.go b/server/start.go index 70aaa7c4e368..4bf74e21805e 100644 --- a/server/start.go +++ b/server/start.go @@ -83,7 +83,7 @@ which accepts a path for the resulting pprof file. cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file") cmd.Flags().String(flagPruning, "syncable", "Pruning strategy: syncable, nothing, everything, custom") cmd.Flags().Int64(flagPruningKeepEvery, 0, "Define the state number that will be kept. Ignored if pruning is not custom.") - cmd.Flags().Int64(flagPruningSnapshotEvery, 0, "Defines the state that will be snapshot for pruning. Ignored if pruning is not custom.") + cmd.Flags().Int64(flagPruningSnapshotEvery, 0, "Defines the state that will be snapshot for state sync and pruning. Ignored if pruning is not custom.") cmd.Flags().String( FlagMinGasPrices, "", "Minimum gas prices to accept for transactions; Any fee in a tx must meet this minimum (e.g. 0.01photino;0.0001stake)", diff --git a/snapshots/alias.go b/snapshots/alias.go new file mode 100644 index 000000000000..9b292f547733 --- /dev/null +++ b/snapshots/alias.go @@ -0,0 +1,20 @@ +package snapshots + +import "github.com/cosmos/cosmos-sdk/snapshots/types" + +const ( + CurrentFormat = types.CurrentFormat +) + +var ( + ErrInvalidMetadata = types.ErrInvalidMetadata + ErrChunkHashMismatch = types.ErrChunkHashMismatch + ErrUnknownFormat = types.ErrUnknownFormat + SnapshotFromABCI = types.SnapshotFromABCI +) + +type ( + Snapshotter = types.Snapshotter + Snapshot = types.Snapshot + Metadata = types.Metadata +) diff --git a/snapshots/helpers_test.go b/snapshots/helpers_test.go new file mode 100644 index 000000000000..061d9c12a868 --- /dev/null +++ b/snapshots/helpers_test.go @@ -0,0 +1,144 @@ +package snapshots_test + +import ( + "bytes" + "crypto/sha256" + "errors" + "io" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + db "github.com/tendermint/tm-db" + + "github.com/cosmos/cosmos-sdk/snapshots" +) + +func checksum(b []byte) []byte { + hash := sha256.Sum256(b) + return hash[:] +} + +func checksums(slice [][]byte) [][]byte { + checksums := [][]byte{} + for _, chunk := range slice { + checksums = append(checksums, checksum(chunk)) + } + return checksums +} + +func hash(chunks [][]byte) []byte { + hasher := sha256.New() + for _, chunk := range chunks { + hasher.Write(chunk) + } + return hasher.Sum(nil) +} + +func makeChunks(chunks [][]byte) <-chan io.ReadCloser { + ch := make(chan io.ReadCloser, len(chunks)) + for _, chunk := range chunks { + ch <- ioutil.NopCloser(bytes.NewReader(chunk)) + } + close(ch) + return ch +} + +func readChunks(chunks <-chan io.ReadCloser) [][]byte { + bodies := [][]byte{} + for chunk := range chunks { + body, err := ioutil.ReadAll(chunk) + if err != nil { + panic(err) + } + bodies = append(bodies, body) + } + return bodies +} + +type mockSnapshotter struct { + chunks [][]byte +} + +func (m *mockSnapshotter) Restore(height uint64, format uint32, chunks <-chan io.ReadCloser) error { + if format == 0 { + return snapshots.ErrUnknownFormat + } + if m.chunks != nil { + return errors.New("already has contents") + } + + m.chunks = [][]byte{} + for reader := range chunks { + chunk, err := ioutil.ReadAll(reader) + if err != nil { + return err + } + m.chunks = append(m.chunks, chunk) + } + + return nil +} + +func (m *mockSnapshotter) Snapshot(height uint64, format uint32) (<-chan io.ReadCloser, error) { + if format == 0 { + return nil, snapshots.ErrUnknownFormat + } + ch := make(chan io.ReadCloser, len(m.chunks)) + for _, chunk := range m.chunks { + ch <- ioutil.NopCloser(bytes.NewReader(chunk)) + } + close(ch) + return ch, nil +} + +// setupBusyManager creates a manager with an empty store that is busy taking a snapshot at height 1. +// The snapshot will complete when the returned closer is called. +func setupBusyManager(t *testing.T) (*snapshots.Manager, func()) { + tempdir, err := ioutil.TempDir("", "") + require.NoError(t, err) + store, err := snapshots.NewStore(db.NewMemDB(), tempdir) + require.NoError(t, err) + hung := newHungSnapshotter() + mgr := snapshots.NewManager(store, hung) + + go func() { + _, err := mgr.Take(1) + require.NoError(t, err) + }() + time.Sleep(10 * time.Millisecond) + + closer := func() { + hung.Close() + os.RemoveAll(tempdir) + } + return mgr, closer +} + +// hungSnapshotter can be used to test operations in progress. Call close to end the snapshot. +type hungSnapshotter struct { + ch chan struct{} +} + +func newHungSnapshotter() *hungSnapshotter { + return &hungSnapshotter{ + ch: make(chan struct{}), + } +} + +func (m *hungSnapshotter) Close() { + close(m.ch) +} + +func (m *hungSnapshotter) Snapshot(height uint64, format uint32) (<-chan io.ReadCloser, error) { + <-m.ch + ch := make(chan io.ReadCloser, 1) + ch <- ioutil.NopCloser(bytes.NewReader([]byte{})) + return ch, nil +} + +func (m *hungSnapshotter) Restore(height uint64, format uint32, chunks <-chan io.ReadCloser) error { + panic("not implemented") +} diff --git a/snapshots/manager.go b/snapshots/manager.go new file mode 100644 index 000000000000..cb3de6cd717a --- /dev/null +++ b/snapshots/manager.go @@ -0,0 +1,223 @@ +package snapshots + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/ioutil" + "sync" + "time" +) + +const ( + opNone operation = "" + opSnapshot operation = "snapshot" + opPrune operation = "prune" + opRestore operation = "restore" + + chunkBufferSize = 4 +) + +// operation represents a Manager operation. Only one operation can be in progress at a time. +type operation string + +// Manager manages snapshot and restore operations for an app, making sure only a single +// long-running operation is in progress at any given time, and provides convenience methods +// mirroring the ABCI interface. +type Manager struct { + store *Store + target Snapshotter + + mtx sync.Mutex + operation operation + chRestore chan<- io.ReadCloser + chRestoreDone <-chan error + restorePending [][]byte // pending chunk hashes +} + +// NewManager creates a new manager. +func NewManager(store *Store, target Snapshotter) *Manager { + return &Manager{ + store: store, + target: target, + } +} + +// begin starts an operation, or errors if one is in progress. It manages the mutex itself. +func (m *Manager) begin(op operation) error { + m.mtx.Lock() + defer m.mtx.Unlock() + return m.beginLocked(op) +} + +// beginLocked begins an operation while already holding the mutex. +func (m *Manager) beginLocked(op operation) error { + if op == opNone { + return errors.New("can't begin a none operation") + } + if m.operation != opNone { + return fmt.Errorf("a %v operation is in progress", op) + } + m.operation = op + return nil +} + +// end ends the current operation. +func (m *Manager) end() { + m.mtx.Lock() + defer m.mtx.Unlock() + m.endLocked() +} + +// endLocked ends the current operation while already holding the mutex. +func (m *Manager) endLocked() { + m.operation = opNone + if m.chRestore != nil { + close(m.chRestore) + m.chRestore = nil + } + m.chRestoreDone = nil + m.restorePending = nil +} + +// List lists snapshots, mirroring ABCI ListSnapshots. It can be concurrent with other operations. +func (m *Manager) List() ([]*Snapshot, error) { + return m.store.List() +} + +// LoadChunk loads a chunk into a byte slice, mirroring ABCI LoadChunk. It can be called +// concurrently with other operations. If the chunk does not exist, nil is returned. +func (m *Manager) LoadChunk(height uint64, format uint32, chunk uint32) ([]byte, error) { + reader, err := m.store.LoadChunk(height, format, chunk) + if err != nil { + return nil, err + } + if reader == nil { + return nil, nil + } + defer reader.Close() + + return ioutil.ReadAll(reader) +} + +// Take takes a snapshot and returns its metadata. +func (m *Manager) Take(height uint64) (*Snapshot, error) { + if m == nil { + return nil, errors.New("no snapshot store configured") + } + err := m.begin(opSnapshot) + if err != nil { + return nil, err + } + defer m.end() + + latest, err := m.store.GetLatest() + if err != nil { + return nil, fmt.Errorf("failed to examine latest snapshot: %w", err) + } + if latest != nil && latest.Height >= height { + return nil, fmt.Errorf("a more recent snapshot already exists at height %v", latest.Height) + } + + chunks, err := m.target.Snapshot(height, CurrentFormat) + if err != nil { + return nil, err + } + return m.store.Save(height, CurrentFormat, chunks) +} + +// Prune prunes snapshots, if no other operations are in progress. +func (m *Manager) Prune(retain uint32) (uint64, error) { + err := m.begin(opPrune) + if err != nil { + return 0, err + } + defer m.end() + return m.store.Prune(retain) +} + +// Restore begins an async snapshot restoration, mirroring ABCI OfferSnapshot. Chunks must be fed +// via RestoreChunk() until the restore is complete or a chunk fails. +func (m *Manager) Restore(snapshot Snapshot) error { + if snapshot.Chunks == 0 { + return fmt.Errorf("%w: no chunks", ErrInvalidMetadata) + } + if uint32(len(snapshot.Metadata.ChunkHashes)) != snapshot.Chunks { + return fmt.Errorf("%w: snapshot has %v chunk hashes, but %v chunks", + ErrInvalidMetadata, + uint32(len(snapshot.Metadata.ChunkHashes)), + snapshot.Chunks) + } + m.mtx.Lock() + defer m.mtx.Unlock() + err := m.beginLocked(opRestore) + if err != nil { + return err + } + + // Start an asynchronous snapshot restoration, passing chunks and completion status via channels. + chChunks := make(chan io.ReadCloser, chunkBufferSize) + chDone := make(chan error, 1) + go func() { + chDone <- m.target.Restore(snapshot.Height, snapshot.Format, chChunks) + close(chDone) + }() + + // Check for any initial errors from the restore, before any chunks are fed. + select { + case err := <-chDone: + if err == nil { + err = errors.New("restore ended unexpectedly") + } + m.endLocked() + return err + case <-time.After(20 * time.Millisecond): + } + + m.chRestore = chChunks + m.chRestoreDone = chDone + m.restorePending = snapshot.Metadata.ChunkHashes + return nil +} + +// RestoreChunk adds a chunk to an active snapshot restoration, mirring ABCI ApplySnapshotChunk. +// Chunks must be given until the restore is complete, returning true, or a chunk errors. +func (m *Manager) RestoreChunk(chunk []byte) (bool, error) { + m.mtx.Lock() + defer m.mtx.Unlock() + if m.operation != opRestore { + return false, fmt.Errorf("no restore operation in progress") + } + + // Check if any errors have occurred yet. + select { + case err := <-m.chRestoreDone: + if err == nil { + err = errors.New("restore ended unexpectedly") + } + m.endLocked() + return false, err + default: + } + + // Verify the chunk hash. + hash := sha256.Sum256(chunk) + if !bytes.Equal(hash[:], m.restorePending[0]) { + return false, fmt.Errorf("%w (expected %x, got %x)", ErrChunkHashMismatch, hash, m.restorePending[0]) + } + + // Pass the chunk to the restore, and wait for completion if it was the final one. + m.chRestore <- ioutil.NopCloser(bytes.NewReader(chunk)) + m.restorePending = m.restorePending[1:] + + if len(m.restorePending) == 0 { + close(m.chRestore) + m.chRestore = nil + err := <-m.chRestoreDone + m.endLocked() + return true, err + } + return false, nil +} diff --git a/snapshots/manager_test.go b/snapshots/manager_test.go new file mode 100644 index 000000000000..6be64a6c84e9 --- /dev/null +++ b/snapshots/manager_test.go @@ -0,0 +1,217 @@ +package snapshots_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cosmos/cosmos-sdk/snapshots" +) + +func TestManager_List(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + manager := snapshots.NewManager(store, nil) + + mgrList, err := manager.List() + require.NoError(t, err) + storeList, err := store.List() + require.NoError(t, err) + + require.NotEmpty(t, storeList) + assert.Equal(t, storeList, mgrList) + + // list should not block or error on busy managers + manager, teardown = setupBusyManager(t) + defer teardown() + list, err := manager.List() + require.NoError(t, err) + assert.Equal(t, []*snapshots.Snapshot{}, list) +} + +func TestManager_LoadChunk(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + manager := snapshots.NewManager(store, nil) + + // Existing chunk should return body + chunk, err := manager.LoadChunk(2, 1, 1) + require.NoError(t, err) + assert.Equal(t, []byte{2, 1, 1}, chunk) + + // Missing chunk should return nil + chunk, err = manager.LoadChunk(2, 1, 9) + require.NoError(t, err) + assert.Nil(t, chunk) + + // LoadChunk should not block or error on busy managers + manager, teardown = setupBusyManager(t) + defer teardown() + chunk, err = manager.LoadChunk(2, 1, 0) + require.NoError(t, err) + assert.Nil(t, chunk) +} + +func TestManager_Take(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + snapshotter := &mockSnapshotter{ + chunks: [][]byte{ + {1, 2, 3}, + {4, 5, 6}, + {7, 8, 9}, + }, + } + manager := snapshots.NewManager(store, snapshotter) + + // nil manager should return error + _, err := (*snapshots.Manager)(nil).Take(1) + require.Error(t, err) + + // taking a snapshot at a lower height than the latest should error + _, err = manager.Take(3) + require.Error(t, err) + + // taking a snapshot at a higher height should be fine, and should return it + snapshot, err := manager.Take(5) + require.NoError(t, err) + assert.Equal(t, &snapshots.Snapshot{ + Height: 5, + Format: snapshots.CurrentFormat, + Chunks: 3, + Hash: []uint8{0x47, 0xe4, 0xee, 0x7f, 0x21, 0x1f, 0x73, 0x26, 0x5d, 0xd1, 0x76, 0x58, 0xf6, 0xe2, 0x1c, 0x13, 0x18, 0xbd, 0x6c, 0x81, 0xf3, 0x75, 0x98, 0xe2, 0xa, 0x27, 0x56, 0x29, 0x95, 0x42, 0xef, 0xcf}, + Metadata: snapshots.Metadata{ + ChunkHashes: [][]byte{ + checksum([]byte{1, 2, 3}), + checksum([]byte{4, 5, 6}), + checksum([]byte{7, 8, 9}), + }, + }, + }, snapshot) + + storeSnapshot, chunks, err := store.Load(snapshot.Height, snapshot.Format) + require.NoError(t, err) + assert.Equal(t, snapshot, storeSnapshot) + assert.Equal(t, [][]byte{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}, readChunks(chunks)) + + // taking a snapshot while a different snapshot is being taken should error + manager, teardown = setupBusyManager(t) + defer teardown() + _, err = manager.Take(9) + require.Error(t, err) +} + +func TestManager_Prune(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + manager := snapshots.NewManager(store, nil) + + pruned, err := manager.Prune(2) + require.NoError(t, err) + assert.EqualValues(t, 1, pruned) + + list, err := manager.List() + require.NoError(t, err) + assert.Len(t, list, 3) + + // Prune should error while a snapshot is being taken + manager, teardown = setupBusyManager(t) + defer teardown() + _, err = manager.Prune(2) + require.Error(t, err) +} + +func TestManager_Restore(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + target := &mockSnapshotter{} + manager := snapshots.NewManager(store, target) + + chunks := [][]byte{ + {1, 2, 3}, + {4, 5, 6}, + {7, 8, 9}, + } + + // Restore errors on invalid format + err := manager.Restore(snapshots.Snapshot{ + Height: 3, + Format: 0, + Hash: []byte{1, 2, 3}, + Chunks: uint32(len(chunks)), + Metadata: snapshots.Metadata{ChunkHashes: checksums(chunks)}, + }) + require.Error(t, err) + require.Equal(t, err, snapshots.ErrUnknownFormat) + + // Restore errors on no chunks + err = manager.Restore(snapshots.Snapshot{Height: 3, Format: 1, Hash: []byte{1, 2, 3}}) + require.Error(t, err) + + // Restore errors on chunk and chunkhashes mismatch + err = manager.Restore(snapshots.Snapshot{ + Height: 3, + Format: 1, + Hash: []byte{1, 2, 3}, + Chunks: 4, + Metadata: snapshots.Metadata{ChunkHashes: checksums(chunks)}, + }) + require.Error(t, err) + + // Starting a restore works + err = manager.Restore(snapshots.Snapshot{ + Height: 3, + Format: 1, + Hash: []byte{1, 2, 3}, + Chunks: 3, + Metadata: snapshots.Metadata{ChunkHashes: checksums(chunks)}, + }) + require.NoError(t, err) + + // While the restore is in progress, any other operations fail + _, err = manager.Take(4) + require.Error(t, err) + + // Feeding the chunks should work + for i, chunk := range chunks { + done, err := manager.RestoreChunk(chunk) + require.NoError(t, err) + if i == len(chunks)-1 { + assert.True(t, done) + } else { + assert.False(t, done) + } + } + + assert.Equal(t, chunks, target.chunks) + + // Starting a new restore should fail now, because the target already has contents. + err = manager.Restore(snapshots.Snapshot{ + Height: 3, + Format: 1, + Hash: []byte{1, 2, 3}, + Chunks: 3, + Metadata: snapshots.Metadata{ChunkHashes: checksums(chunks)}, + }) + require.Error(t, err) + + // But if we clear out the target we should be able to start a new restore. This time we'll + // fail it with a checksum error. That error should stop the operation, so that we can do + // a prune operation right after. + target.chunks = nil + err = manager.Restore(snapshots.Snapshot{ + Height: 3, + Format: 1, + Hash: []byte{1, 2, 3}, + Chunks: 3, + Metadata: snapshots.Metadata{ChunkHashes: checksums(chunks)}, + }) + require.NoError(t, err) + + _, err = manager.RestoreChunk([]byte{9, 9, 9}) + require.Error(t, err) + + _, err = manager.Prune(1) + require.NoError(t, err) +} diff --git a/snapshots/store.go b/snapshots/store.go new file mode 100644 index 000000000000..b566f0b645e0 --- /dev/null +++ b/snapshots/store.go @@ -0,0 +1,370 @@ +package snapshots + +import ( + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "strconv" + "sync" + + "github.com/gogo/protobuf/proto" + db "github.com/tendermint/tm-db" + + "github.com/cosmos/cosmos-sdk/snapshots/types" +) + +const ( + // keyPrefixSnapshot is the prefix for snapshot database keys + keyPrefixSnapshot byte = 0x01 +) + +// Store is a snapshot store, containing snapshot metadata and binary chunks. +type Store struct { + db db.DB + dir string + + mtx sync.Mutex + saving map[uint64]bool // heights currently being saved +} + +// NewStore creates a new snapshot store. +func NewStore(db db.DB, dir string) (*Store, error) { + if dir == "" { + return nil, errors.New("snapshot directory not given") + } + err := os.MkdirAll(dir, 0755) + if err != nil { + return nil, fmt.Errorf("failed to create snapshot directory %q: %w", dir, err) + } + + return &Store{ + db: db, + dir: dir, + saving: make(map[uint64]bool), + }, nil +} + +// Delete deletes a snapshot. +func (s *Store) Delete(height uint64, format uint32) error { + s.mtx.Lock() + saving := s.saving[height] + s.mtx.Unlock() + if saving { + return fmt.Errorf("snapshot for height %v format %v is currently being saved", height, format) + } + err := s.db.DeleteSync(encodeKey(height, format)) + if err != nil { + return fmt.Errorf("failed to delete snapshot for height %v format %v: %w", + height, format, err) + } + err = os.RemoveAll(s.pathSnapshot(height, format)) + if err != nil { + return fmt.Errorf("failed to delete snapshot chunks for height %v format %v: %w", + height, format, err) + } + return nil +} + +// Get fetches snapshot info from the database. +func (s *Store) Get(height uint64, format uint32) (*types.Snapshot, error) { + bytes, err := s.db.Get(encodeKey(height, format)) + if err != nil { + return nil, fmt.Errorf("failed to fetch snapshot metadata for height %v format %v: %w", + height, format, err) + } + if bytes == nil { + return nil, nil + } + snapshot := &types.Snapshot{} + err = proto.Unmarshal(bytes, snapshot) + if err != nil { + return nil, fmt.Errorf("failed to decode snapshot metadata for height %v format %v: %w", + height, format, err) + } + if snapshot.Metadata.ChunkHashes == nil { + snapshot.Metadata.ChunkHashes = [][]byte{} + } + return snapshot, nil +} + +// Get fetches the latest snapshot from the database, if any. +func (s *Store) GetLatest() (*types.Snapshot, error) { + iter, err := s.db.ReverseIterator(encodeKey(0, 0), encodeKey(math.MaxUint64, math.MaxUint32)) + if err != nil { + return nil, fmt.Errorf("failed to find latest snapshot: %w", err) + } + defer iter.Close() + + var snapshot *types.Snapshot + if iter.Valid() { + snapshot = &types.Snapshot{} + err := proto.Unmarshal(iter.Value(), snapshot) + if err != nil { + return nil, fmt.Errorf("failed to decode latest snapshot: %w", err) + } + } + err = iter.Error() + if err != nil { + return nil, fmt.Errorf("failed to find latest snapshot: %w", err) + } + return snapshot, nil +} + +// List lists snapshots, in reverse order (newest first). +func (s *Store) List() ([]*types.Snapshot, error) { + iter, err := s.db.ReverseIterator(encodeKey(0, 0), encodeKey(math.MaxUint64, math.MaxUint32)) + if err != nil { + return nil, fmt.Errorf("failed to list snapshots: %w", err) + } + defer iter.Close() + + snapshots := make([]*types.Snapshot, 0) + for ; iter.Valid(); iter.Next() { + snapshot := &types.Snapshot{} + err := proto.Unmarshal(iter.Value(), snapshot) + if err != nil { + return nil, fmt.Errorf("failed to decode snapshot info: %w", err) + } + snapshots = append(snapshots, snapshot) + } + err = iter.Error() + if err != nil { + return nil, err + } + return snapshots, nil +} + +// Load loads a snapshot (both metadata and binary chunks). The chunks must be consumed and closed. +// Returns nil if the snapshot does not exist. +func (s *Store) Load(height uint64, format uint32) (*types.Snapshot, <-chan io.ReadCloser, error) { + snapshot, err := s.Get(height, format) + if err != nil { + return nil, nil, err + } + if snapshot == nil { + return nil, nil, nil + } + + ch := make(chan io.ReadCloser) + go func() { + defer close(ch) + for i := uint32(0); i < snapshot.Chunks; i++ { + pr, pw := io.Pipe() + ch <- pr + chunk, err := s.loadChunkFile(height, format, i) + if err != nil { + pw.CloseWithError(err) + return + } + defer chunk.Close() + _, err = io.Copy(pw, chunk) + if err != nil { + pw.CloseWithError(err) + return + } + chunk.Close() + pw.Close() + } + }() + + return snapshot, ch, nil +} + +// LoadChunk loads a chunk from disk, or returns nil if it does not exist. The caller must call +// Close() on it when done. +func (s *Store) LoadChunk(height uint64, format uint32, chunk uint32) (io.ReadCloser, error) { + path := s.pathChunk(height, format, chunk) + file, err := os.Open(path) + if os.IsNotExist(err) { + return nil, nil + } + return file, err +} + +// loadChunkFile loads a chunk from disk, and errors if it does not exist. +func (s *Store) loadChunkFile(height uint64, format uint32, chunk uint32) (io.ReadCloser, error) { + path := s.pathChunk(height, format, chunk) + file, err := os.Open(path) + if err != nil { + return nil, err + } + return file, nil +} + +// Prune removes old snapshots. The given number of most recent heights (regardless of format) are retained. +func (s *Store) Prune(retain uint32) (uint64, error) { + iter, err := s.db.ReverseIterator(encodeKey(0, 0), encodeKey(math.MaxUint64, math.MaxUint32)) + if err != nil { + return 0, fmt.Errorf("failed to prune snapshots: %w", err) + } + defer iter.Close() + + pruned := uint64(0) + prunedHeights := make(map[uint64]bool) + skip := make(map[uint64]bool) + for ; iter.Valid(); iter.Next() { + height, format, err := decodeKey(iter.Key()) + if err != nil { + return 0, fmt.Errorf("failed to prune snapshots: %w", err) + } + if skip[height] || uint32(len(skip)) < retain { + skip[height] = true + continue + } + err = s.Delete(height, format) + if err != nil { + return 0, fmt.Errorf("failed to prune snapshots: %w", err) + } + pruned++ + prunedHeights[height] = true + } + // Since Delete() deletes a specific format, while we want to prune a height, we clean up + // the height directory as well + for height, ok := range prunedHeights { + if ok { + err = os.Remove(s.pathHeight(height)) + if err != nil { + return 0, fmt.Errorf("failed to remove snapshot directory for height %v", height) + } + } + } + err = iter.Error() + if err != nil { + return 0, err + } + return pruned, nil +} + +// Save saves a snapshot to disk, returning it. +func (s *Store) Save(height uint64, format uint32, chunks <-chan io.ReadCloser) (*Snapshot, error) { + defer DrainChunks(chunks) + if height == 0 { + return nil, errors.New("snapshot height cannot be 0") + } + + s.mtx.Lock() + saving := s.saving[height] + s.saving[height] = true + s.mtx.Unlock() + if saving { + return nil, fmt.Errorf("a snapshot for height %v is already being saved", height) + } + defer func() { + s.mtx.Lock() + delete(s.saving, height) + s.mtx.Unlock() + }() + + exists, err := s.db.Has(encodeKey(height, format)) + if err != nil { + return nil, err + } + if exists { + return nil, fmt.Errorf("snapshot already exists for height %v format %v", height, format) + } + + snapshot := &types.Snapshot{ + Height: height, + Format: format, + } + index := uint32(0) + snapshotHasher := sha256.New() + for chunkBody := range chunks { + defer chunkBody.Close() // nolint: staticcheck + chunkHasher := sha256.New() + dir := s.pathSnapshot(height, format) + err = os.MkdirAll(dir, 0755) + if err != nil { + return nil, fmt.Errorf("failed to create snapshot directory %q: %w", dir, err) + } + path := s.pathChunk(height, format, index) + file, err := os.Create(path) + if err != nil { + return nil, fmt.Errorf("failed to create snapshot chunk file %q: %w", path, err) + } + defer file.Close() // nolint: staticcheck + _, err = io.Copy(io.MultiWriter(file, chunkHasher, snapshotHasher), chunkBody) + if err != nil { + return nil, fmt.Errorf("failed to generate snapshot chunk %v: %w", index, err) + } + err = file.Close() + if err != nil { + return nil, fmt.Errorf("failed to close snapshot chunk %v: %w", index, err) + } + err = chunkBody.Close() + if err != nil { + return nil, fmt.Errorf("failed to close snapshot chunk %v: %w", index, err) + } + snapshot.Metadata.ChunkHashes = append(snapshot.Metadata.ChunkHashes, chunkHasher.Sum(nil)) + index++ + } + snapshot.Chunks = index + snapshot.Hash = snapshotHasher.Sum(nil) + err = s.saveSnapshot(snapshot) + if err != nil { + return nil, err + } + return snapshot, nil +} + +// saveSnapshot saves snapshot metadata to the database. +func (s *Store) saveSnapshot(snapshot *types.Snapshot) error { + value, err := proto.Marshal(snapshot) + if err != nil { + return fmt.Errorf("failed to encode snapshot metadata: %w", err) + } + err = s.db.SetSync(encodeKey(snapshot.Height, snapshot.Format), value) + if err != nil { + return fmt.Errorf("failed to store snapshot: %w", err) + } + return nil +} + +// pathHeight generates the path to a height, containing multiple snapshot formats. +func (s *Store) pathHeight(height uint64) string { + return filepath.Join(s.dir, strconv.FormatUint(height, 10)) +} + +// pathSnapshot generates a snapshot path, as a specific format under a height. +func (s *Store) pathSnapshot(height uint64, format uint32) string { + return filepath.Join(s.pathHeight(height), strconv.FormatUint(uint64(format), 10)) +} + +// pathChunk generates a snapshot chunk path. +func (s *Store) pathChunk(height uint64, format uint32, chunk uint32) string { + return filepath.Join(s.pathSnapshot(height, format), strconv.FormatUint(uint64(chunk), 10)) +} + +// decodeKey decodes a snapshot key. +func decodeKey(k []byte) (uint64, uint32, error) { + if len(k) != 13 { + return 0, 0, fmt.Errorf("invalid snapshot key with length %v", len(k)) + } + if k[0] != keyPrefixSnapshot { + return 0, 0, fmt.Errorf("invalid snapshot key prefix %x", k[0]) + } + height := binary.BigEndian.Uint64(k[1:9]) + format := binary.BigEndian.Uint32(k[9:13]) + return height, format, nil +} + +// encodeKey encodes a snapshot key. +func encodeKey(height uint64, format uint32) []byte { + k := make([]byte, 0, 13) + k = append(k, keyPrefixSnapshot) + + bHeight := make([]byte, 8) + binary.BigEndian.PutUint64(bHeight, height) + k = append(k, bHeight...) + + bFormat := make([]byte, 4) + binary.BigEndian.PutUint32(bFormat, format) + k = append(k, bFormat...) + + return k +} diff --git a/snapshots/store_test.go b/snapshots/store_test.go new file mode 100644 index 000000000000..7d9e0bba7fa1 --- /dev/null +++ b/snapshots/store_test.go @@ -0,0 +1,366 @@ +package snapshots_test + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + db "github.com/tendermint/tm-db" + + "github.com/cosmos/cosmos-sdk/snapshots" + "github.com/cosmos/cosmos-sdk/snapshots/types" +) + +func setupStore(t *testing.T) (*snapshots.Store, func()) { + tempdir, err := ioutil.TempDir("", "snapshots") + require.NoError(t, err) + + store, err := snapshots.NewStore(db.NewMemDB(), tempdir) + require.NoError(t, err) + + _, err = store.Save(1, 1, makeChunks([][]byte{ + {1, 1, 0}, {1, 1, 1}, + })) + require.NoError(t, err) + _, err = store.Save(2, 1, makeChunks([][]byte{ + {2, 1, 0}, {2, 1, 1}, + })) + require.NoError(t, err) + _, err = store.Save(2, 2, makeChunks([][]byte{ + {2, 2, 0}, {2, 2, 1}, {2, 2, 2}, + })) + require.NoError(t, err) + _, err = store.Save(3, 2, makeChunks([][]byte{ + {3, 2, 0}, {3, 2, 1}, {3, 2, 2}, + })) + require.NoError(t, err) + + teardown := func() { + err := os.RemoveAll(tempdir) + if err != nil { + t.Logf("Failed to remove tempdir %q: %v", tempdir, err) + } + } + return store, teardown +} + +func TestNewStore(t *testing.T) { + tempdir, err := ioutil.TempDir("", "snapshots") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + _, err = snapshots.NewStore(db.NewMemDB(), tempdir) + require.NoError(t, err) +} + +func TestNewStore_ErrNoDir(t *testing.T) { + _, err := snapshots.NewStore(db.NewMemDB(), "") + require.Error(t, err) +} + +func TestNewStore_ErrDirFailure(t *testing.T) { + tempfile, err := ioutil.TempFile("", "snapshots") + require.NoError(t, err) + defer func() { + os.RemoveAll(tempfile.Name()) + tempfile.Close() + }() + tempdir := filepath.Join(tempfile.Name(), "subdir") + + _, err = snapshots.NewStore(db.NewMemDB(), tempdir) + require.Error(t, err) +} + +func TestStore_Delete(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + + // Deleting a snapshot should remove it + err := store.Delete(2, 2) + require.NoError(t, err) + + snapshot, err := store.Get(2, 2) + require.NoError(t, err) + assert.Nil(t, snapshot) + + snapshots, err := store.List() + require.NoError(t, err) + assert.Len(t, snapshots, 3) + + // Deleting it again should not error + err = store.Delete(2, 2) + require.NoError(t, err) + + // Deleting a snapshot being saved should error + ch := make(chan io.ReadCloser) + go store.Save(9, 1, ch) + + time.Sleep(10 * time.Millisecond) + err = store.Delete(9, 1) + require.Error(t, err) + + // But after it's saved it should work + close(ch) + time.Sleep(10 * time.Millisecond) + err = store.Delete(9, 1) + require.NoError(t, err) +} + +func TestStore_Get(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + + // Loading a missing snapshot should return nil + snapshot, err := store.Get(9, 9) + require.NoError(t, err) + assert.Nil(t, snapshot) + + // Loading a snapshot should returns its metadata + snapshot, err = store.Get(2, 1) + require.NoError(t, err) + assert.Equal(t, &types.Snapshot{ + Height: 2, + Format: 1, + Chunks: 2, + Hash: hash([][]byte{{2, 1, 0}, {2, 1, 1}}), + Metadata: types.Metadata{ + ChunkHashes: [][]byte{ + checksum([]byte{2, 1, 0}), + checksum([]byte{2, 1, 1}), + }, + }, + }, snapshot) +} + +func TestStore_GetLatest(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + + // Loading a missing snapshot should return nil + snapshot, err := store.GetLatest() + require.NoError(t, err) + assert.Equal(t, &types.Snapshot{ + Height: 3, + Format: 2, + Chunks: 3, + Hash: hash([][]byte{ + {3, 2, 0}, + {3, 2, 1}, + {3, 2, 2}, + }), + Metadata: types.Metadata{ + ChunkHashes: checksums([][]byte{ + {3, 2, 0}, + {3, 2, 1}, + {3, 2, 2}, + }), + }, + }, snapshot) +} + +func TestStore_List(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + + snapshots, err := store.List() + require.NoError(t, err) + + require.Equal(t, []*types.Snapshot{ + {Height: 3, Format: 2, Chunks: 3, Hash: hash([][]byte{{3, 2, 0}, {3, 2, 1}, {3, 2, 2}}), + Metadata: types.Metadata{ChunkHashes: checksums([][]byte{{3, 2, 0}, {3, 2, 1}, {3, 2, 2}})}, + }, + {Height: 2, Format: 2, Chunks: 3, Hash: hash([][]byte{{2, 2, 0}, {2, 2, 1}, {2, 2, 2}}), + Metadata: types.Metadata{ChunkHashes: checksums([][]byte{{2, 2, 0}, {2, 2, 1}, {2, 2, 2}})}, + }, + {Height: 2, Format: 1, Chunks: 2, Hash: hash([][]byte{{2, 1, 0}, {2, 1, 1}}), + Metadata: types.Metadata{ChunkHashes: checksums([][]byte{{2, 1, 0}, {2, 1, 1}})}, + }, + {Height: 1, Format: 1, Chunks: 2, Hash: hash([][]byte{{1, 1, 0}, {1, 1, 1}}), + Metadata: types.Metadata{ChunkHashes: checksums([][]byte{{1, 1, 0}, {1, 1, 1}})}, + }, + }, snapshots) +} + +func TestStore_Load(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + + // Loading a missing snapshot should return nil + snapshot, chunks, err := store.Load(9, 9) + require.NoError(t, err) + assert.Nil(t, snapshot) + assert.Nil(t, chunks) + + // Loading a snapshot should returns its metadata and chunks + snapshot, chunks, err = store.Load(2, 1) + require.NoError(t, err) + assert.Equal(t, &types.Snapshot{ + Height: 2, + Format: 1, + Chunks: 2, + Hash: hash([][]byte{{2, 1, 0}, {2, 1, 1}}), + Metadata: types.Metadata{ + ChunkHashes: [][]byte{ + checksum([]byte{2, 1, 0}), + checksum([]byte{2, 1, 1}), + }, + }, + }, snapshot) + + for i := uint32(0); i < snapshot.Chunks; i++ { + reader, ok := <-chunks + require.True(t, ok) + chunk, err := ioutil.ReadAll(reader) + require.NoError(t, err) + err = reader.Close() + require.NoError(t, err) + assert.Equal(t, []byte{2, 1, byte(i)}, chunk) + } + assert.Empty(t, chunks) +} + +func TestStore_LoadChunk(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + + // Loading a missing snapshot should return nil + chunk, err := store.LoadChunk(9, 9, 0) + require.NoError(t, err) + assert.Nil(t, chunk) + + // Loading a missing chunk index should return nil + chunk, err = store.LoadChunk(2, 1, 2) + require.NoError(t, err) + require.Nil(t, chunk) + + // Loading a chunk should returns a content reader + chunk, err = store.LoadChunk(2, 1, 0) + require.NoError(t, err) + require.NotNil(t, chunk) + body, err := ioutil.ReadAll(chunk) + require.NoError(t, err) + assert.Equal(t, []byte{2, 1, 0}, body) + err = chunk.Close() + require.NoError(t, err) +} + +func TestStore_Prune(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + + // Pruning too many snapshots should be fine + pruned, err := store.Prune(4) + require.NoError(t, err) + assert.EqualValues(t, 0, pruned) + + snapshots, err := store.List() + require.NoError(t, err) + assert.Len(t, snapshots, 4) + + // Pruning until the last two heights should leave three snapshots (for two heights) + pruned, err = store.Prune(2) + require.NoError(t, err) + assert.EqualValues(t, 1, pruned) + + snapshots, err = store.List() + require.NoError(t, err) + require.Equal(t, []*types.Snapshot{ + {Height: 3, Format: 2, Chunks: 3, Hash: hash([][]byte{{3, 2, 0}, {3, 2, 1}, {3, 2, 2}}), + Metadata: types.Metadata{ChunkHashes: checksums([][]byte{{3, 2, 0}, {3, 2, 1}, {3, 2, 2}})}, + }, + {Height: 2, Format: 2, Chunks: 3, Hash: hash([][]byte{{2, 2, 0}, {2, 2, 1}, {2, 2, 2}}), + Metadata: types.Metadata{ChunkHashes: checksums([][]byte{{2, 2, 0}, {2, 2, 1}, {2, 2, 2}})}, + }, + {Height: 2, Format: 1, Chunks: 2, Hash: hash([][]byte{{2, 1, 0}, {2, 1, 1}}), + Metadata: types.Metadata{ChunkHashes: checksums([][]byte{{2, 1, 0}, {2, 1, 1}})}, + }, + }, snapshots) + + // Pruning all heights should also be fine + pruned, err = store.Prune(0) + require.NoError(t, err) + assert.EqualValues(t, 3, pruned) + + snapshots, err = store.List() + require.NoError(t, err) + assert.Empty(t, snapshots) +} + +func TestStore_Save(t *testing.T) { + store, teardown := setupStore(t) + defer teardown() + + // Saving a snapshot should work + snapshot, err := store.Save(4, 1, makeChunks([][]byte{{1}, {2}})) + require.NoError(t, err) + assert.Equal(t, &snapshots.Snapshot{ + Height: 4, + Format: 1, + Chunks: 2, + Hash: hash([][]byte{{1}, {2}}), + Metadata: snapshots.Metadata{ + ChunkHashes: [][]byte{ + checksum([]byte{1}), + checksum([]byte{2}), + }, + }, + }, snapshot) + loaded, err := store.Get(snapshot.Height, snapshot.Format) + require.NoError(t, err) + assert.Equal(t, snapshot, loaded) + + // Saving an existing snapshot should error + _, err = store.Save(4, 1, makeChunks([][]byte{{1}, {2}})) + require.Error(t, err) + + // Saving at height 0 should error + _, err = store.Save(0, 1, makeChunks([][]byte{{1}, {2}})) + require.Error(t, err) + + // Saving at format 0 should be fine + _, err = store.Save(1, 0, makeChunks([][]byte{{1}, {2}})) + require.NoError(t, err) + + // Saving a snapshot with no chunks should be fine, as should loading it + _, err = store.Save(5, 1, makeChunks([][]byte{})) + require.NoError(t, err) + snapshot, chunks, err := store.Load(5, 1) + require.NoError(t, err) + assert.Equal(t, &types.Snapshot{Height: 5, Format: 1, Hash: hash([][]byte{}), Metadata: types.Metadata{ChunkHashes: [][]byte{}}}, snapshot) + assert.Empty(t, chunks) + + // Saving a snapshot should error if a chunk reader returns an error, and it should empty out + // the channel + someErr := errors.New("boom") + pr, pw := io.Pipe() + err = pw.CloseWithError(someErr) + require.NoError(t, err) + + ch := make(chan io.ReadCloser, 2) + ch <- pr + ch <- ioutil.NopCloser(bytes.NewBuffer([]byte{0xff})) + close(ch) + + _, err = store.Save(6, 1, ch) + require.Error(t, err) + require.True(t, errors.Is(err, someErr)) + assert.Empty(t, ch) + + // Saving a snapshot should error if a snapshot is already in progress for the same height, + // regardless of format. However, a different height should succeed. + ch = make(chan io.ReadCloser) + go store.Save(7, 1, ch) + time.Sleep(10 * time.Millisecond) + _, err = store.Save(7, 2, makeChunks(nil)) + require.Error(t, err) + _, err = store.Save(8, 1, makeChunks(nil)) + require.NoError(t, err) + close(ch) +} diff --git a/snapshots/types/convert.go b/snapshots/types/convert.go new file mode 100644 index 000000000000..33e315ee032c --- /dev/null +++ b/snapshots/types/convert.go @@ -0,0 +1,39 @@ +package types + +import ( + fmt "fmt" + + proto "github.com/gogo/protobuf/proto" + abci "github.com/tendermint/tendermint/abci/types" +) + +// Converts an ABCI snapshot to a snapshot. Mainly to decode the SDK metadata. +func SnapshotFromABCI(in *abci.Snapshot) (Snapshot, error) { + snapshot := Snapshot{ + Height: in.Height, + Format: in.Format, + Chunks: in.Chunks, + Hash: in.Hash, + } + err := proto.Unmarshal(in.Metadata, &snapshot.Metadata) + if err != nil { + return Snapshot{}, fmt.Errorf("failed to unmarshal snapshot metadata: %w", err) + } + return snapshot, nil +} + +// Converts a Snapshot to its ABCI representation. Mainly to encode the SDK metadata. +func (s Snapshot) ToABCI() (abci.Snapshot, error) { + out := abci.Snapshot{ + Height: s.Height, + Format: s.Format, + Chunks: s.Chunks, + Hash: s.Hash, + } + var err error + out.Metadata, err = proto.Marshal(&s.Metadata) + if err != nil { + return abci.Snapshot{}, fmt.Errorf("failed to marshal snapshot metadata: %w", err) + } + return out, nil +} diff --git a/snapshots/types/errors.go b/snapshots/types/errors.go new file mode 100644 index 000000000000..c98dd055ed88 --- /dev/null +++ b/snapshots/types/errors.go @@ -0,0 +1,16 @@ +package types + +import ( + "errors" +) + +var ( + // ErrUnknownFormat is returned when an unknown format is used. + ErrUnknownFormat = errors.New("unknown snapshot format") + + // ErrChunkHashMismatch is returned when chunk hash verification failed. + ErrChunkHashMismatch = errors.New("chunk hash verification failed") + + // ErrInvalidMetadata is returned when the snapshot metadata is invalid. + ErrInvalidMetadata = errors.New("invalid snapshot metadata") +) diff --git a/snapshots/types/format.go b/snapshots/types/format.go new file mode 100644 index 000000000000..edfdb36d7bfc --- /dev/null +++ b/snapshots/types/format.go @@ -0,0 +1,6 @@ +package types + +// CurrentFormat is the currently used format for snapshots. Snapshots using the same format +// must be identical across all nodes for a given height, so this must be bumped when the binary +// snapshot output changes. +const CurrentFormat uint32 = 1 diff --git a/snapshots/types/snapshotter.go b/snapshots/types/snapshotter.go new file mode 100644 index 000000000000..af9c2bb5bc13 --- /dev/null +++ b/snapshots/types/snapshotter.go @@ -0,0 +1,14 @@ +package types + +import "io" + +// Snapshotter is something that can take and restore snapshots, consisting of streamed binary +// chunks - all of which must be read from the channel and closed. If an unsupported format is +// given, it must return ErrUnknownFormat (possibly wrapped with fmt.Errorf). +type Snapshotter interface { + // Snapshot takes a state snapshot. + Snapshot(height uint64, format uint32) (<-chan io.ReadCloser, error) + + // Restore restores a state snapshot. + Restore(height uint64, format uint32, chunks <-chan io.ReadCloser) error +} diff --git a/snapshots/types/types.pb.go b/snapshots/types/types.pb.go new file mode 100644 index 000000000000..4835f02de6b5 --- /dev/null +++ b/snapshots/types/types.pb.go @@ -0,0 +1,664 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: snapshots/types/types.proto + +package types + +import ( + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +// Snapshot contains snapshot metadata. +type Snapshot struct { + Height uint64 `protobuf:"varint,1,opt,name=height,proto3" json:"height,omitempty"` + Format uint32 `protobuf:"varint,2,opt,name=format,proto3" json:"format,omitempty"` + Chunks uint32 `protobuf:"varint,3,opt,name=chunks,proto3" json:"chunks,omitempty"` + Hash []byte `protobuf:"bytes,4,opt,name=hash,proto3" json:"hash,omitempty"` + Metadata Metadata `protobuf:"bytes,5,opt,name=metadata,proto3" json:"metadata"` +} + +func (m *Snapshot) Reset() { *m = Snapshot{} } +func (m *Snapshot) String() string { return proto.CompactTextString(m) } +func (*Snapshot) ProtoMessage() {} +func (*Snapshot) Descriptor() ([]byte, []int) { + return fileDescriptor_03ed7742cffb8ebd, []int{0} +} +func (m *Snapshot) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Snapshot) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Snapshot.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Snapshot) XXX_Merge(src proto.Message) { + xxx_messageInfo_Snapshot.Merge(m, src) +} +func (m *Snapshot) XXX_Size() int { + return m.Size() +} +func (m *Snapshot) XXX_DiscardUnknown() { + xxx_messageInfo_Snapshot.DiscardUnknown(m) +} + +var xxx_messageInfo_Snapshot proto.InternalMessageInfo + +func (m *Snapshot) GetHeight() uint64 { + if m != nil { + return m.Height + } + return 0 +} + +func (m *Snapshot) GetFormat() uint32 { + if m != nil { + return m.Format + } + return 0 +} + +func (m *Snapshot) GetChunks() uint32 { + if m != nil { + return m.Chunks + } + return 0 +} + +func (m *Snapshot) GetHash() []byte { + if m != nil { + return m.Hash + } + return nil +} + +func (m *Snapshot) GetMetadata() Metadata { + if m != nil { + return m.Metadata + } + return Metadata{} +} + +type Metadata struct { + ChunkHashes [][]byte `protobuf:"bytes,1,rep,name=chunk_hashes,json=chunkHashes,proto3" json:"chunk_hashes,omitempty"` +} + +func (m *Metadata) Reset() { *m = Metadata{} } +func (m *Metadata) String() string { return proto.CompactTextString(m) } +func (*Metadata) ProtoMessage() {} +func (*Metadata) Descriptor() ([]byte, []int) { + return fileDescriptor_03ed7742cffb8ebd, []int{1} +} +func (m *Metadata) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Metadata) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Metadata.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Metadata) XXX_Merge(src proto.Message) { + xxx_messageInfo_Metadata.Merge(m, src) +} +func (m *Metadata) XXX_Size() int { + return m.Size() +} +func (m *Metadata) XXX_DiscardUnknown() { + xxx_messageInfo_Metadata.DiscardUnknown(m) +} + +var xxx_messageInfo_Metadata proto.InternalMessageInfo + +func (m *Metadata) GetChunkHashes() [][]byte { + if m != nil { + return m.ChunkHashes + } + return nil +} + +func init() { + proto.RegisterType((*Snapshot)(nil), "cosmos_sdk.snapshots.v1.Snapshot") + proto.RegisterType((*Metadata)(nil), "cosmos_sdk.snapshots.v1.Metadata") +} + +func init() { proto.RegisterFile("snapshots/types/types.proto", fileDescriptor_03ed7742cffb8ebd) } + +var fileDescriptor_03ed7742cffb8ebd = []byte{ + // 288 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x2e, 0xce, 0x4b, 0x2c, + 0x28, 0xce, 0xc8, 0x2f, 0x29, 0xd6, 0x2f, 0xa9, 0x2c, 0x48, 0x85, 0x92, 0x7a, 0x05, 0x45, 0xf9, + 0x25, 0xf9, 0x42, 0xe2, 0xc9, 0xf9, 0xc5, 0xb9, 0xf9, 0xc5, 0xf1, 0xc5, 0x29, 0xd9, 0x7a, 0x70, + 0x75, 0x7a, 0x65, 0x86, 0x52, 0x6a, 0x25, 0x19, 0x99, 0x45, 0x29, 0xf1, 0x05, 0x89, 0x45, 0x25, + 0x95, 0xfa, 0x60, 0xb5, 0xfa, 0xe9, 0xf9, 0xe9, 0xf9, 0x08, 0x16, 0xc4, 0x00, 0xa5, 0xd5, 0x8c, + 0x5c, 0x1c, 0xc1, 0x50, 0x8d, 0x42, 0x62, 0x5c, 0x6c, 0x19, 0xa9, 0x99, 0xe9, 0x19, 0x25, 0x12, + 0x8c, 0x0a, 0x8c, 0x1a, 0x2c, 0x41, 0x50, 0x1e, 0x48, 0x3c, 0x2d, 0xbf, 0x28, 0x37, 0xb1, 0x44, + 0x82, 0x49, 0x81, 0x51, 0x83, 0x37, 0x08, 0xca, 0x03, 0x89, 0x27, 0x67, 0x94, 0xe6, 0x65, 0x17, + 0x4b, 0x30, 0x43, 0xc4, 0x21, 0x3c, 0x21, 0x21, 0x2e, 0x96, 0x8c, 0xc4, 0xe2, 0x0c, 0x09, 0x16, + 0x05, 0x46, 0x0d, 0x9e, 0x20, 0x30, 0x5b, 0xc8, 0x99, 0x8b, 0x23, 0x37, 0xb5, 0x24, 0x31, 0x25, + 0xb1, 0x24, 0x51, 0x82, 0x55, 0x81, 0x51, 0x83, 0xdb, 0x48, 0x51, 0x0f, 0x87, 0xe3, 0xf5, 0x7c, + 0xa1, 0x0a, 0x9d, 0x58, 0x4e, 0xdc, 0x93, 0x67, 0x08, 0x82, 0x6b, 0x54, 0xd2, 0xe5, 0xe2, 0x80, + 0xc9, 0x09, 0x29, 0x72, 0xf1, 0x80, 0xad, 0x8b, 0x07, 0x19, 0x9f, 0x5a, 0x2c, 0xc1, 0xa8, 0xc0, + 0xac, 0xc1, 0x13, 0xc4, 0x0d, 0x16, 0xf3, 0x00, 0x0b, 0x39, 0xb9, 0x9d, 0x78, 0x24, 0xc7, 0x78, + 0xe1, 0x91, 0x1c, 0xe3, 0x83, 0x47, 0x72, 0x8c, 0x13, 0x1e, 0xcb, 0x31, 0x5c, 0x78, 0x2c, 0xc7, + 0x70, 0xe3, 0xb1, 0x1c, 0x43, 0x94, 0x4e, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, 0x5e, 0x72, 0x7e, + 0xae, 0x3e, 0xc4, 0x15, 0x50, 0x4a, 0xb7, 0x38, 0x25, 0x5b, 0x1f, 0x2d, 0xc4, 0x93, 0xd8, 0xc0, + 0x61, 0x65, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0x15, 0xba, 0x89, 0x65, 0x8b, 0x01, 0x00, 0x00, +} + +func (m *Snapshot) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Snapshot) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Snapshot) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + { + size, err := m.Metadata.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + if len(m.Hash) > 0 { + i -= len(m.Hash) + copy(dAtA[i:], m.Hash) + i = encodeVarintTypes(dAtA, i, uint64(len(m.Hash))) + i-- + dAtA[i] = 0x22 + } + if m.Chunks != 0 { + i = encodeVarintTypes(dAtA, i, uint64(m.Chunks)) + i-- + dAtA[i] = 0x18 + } + if m.Format != 0 { + i = encodeVarintTypes(dAtA, i, uint64(m.Format)) + i-- + dAtA[i] = 0x10 + } + if m.Height != 0 { + i = encodeVarintTypes(dAtA, i, uint64(m.Height)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *Metadata) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Metadata) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Metadata) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.ChunkHashes) > 0 { + for iNdEx := len(m.ChunkHashes) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.ChunkHashes[iNdEx]) + copy(dAtA[i:], m.ChunkHashes[iNdEx]) + i = encodeVarintTypes(dAtA, i, uint64(len(m.ChunkHashes[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func encodeVarintTypes(dAtA []byte, offset int, v uint64) int { + offset -= sovTypes(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *Snapshot) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Height != 0 { + n += 1 + sovTypes(uint64(m.Height)) + } + if m.Format != 0 { + n += 1 + sovTypes(uint64(m.Format)) + } + if m.Chunks != 0 { + n += 1 + sovTypes(uint64(m.Chunks)) + } + l = len(m.Hash) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + l = m.Metadata.Size() + n += 1 + l + sovTypes(uint64(l)) + return n +} + +func (m *Metadata) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.ChunkHashes) > 0 { + for _, b := range m.ChunkHashes { + l = len(b) + n += 1 + l + sovTypes(uint64(l)) + } + } + return n +} + +func sovTypes(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozTypes(x uint64) (n int) { + return sovTypes(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *Snapshot) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Snapshot: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Snapshot: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Height", wireType) + } + m.Height = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Height |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Format", wireType) + } + m.Format = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Format |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Chunks", wireType) + } + m.Chunks = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Chunks |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Hash = append(m.Hash[:0], dAtA[iNdEx:postIndex]...) + if m.Hash == nil { + m.Hash = []byte{} + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Metadata", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Metadata.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Metadata) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Metadata: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Metadata: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ChunkHashes", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ChunkHashes = append(m.ChunkHashes, make([]byte, postIndex-iNdEx)) + copy(m.ChunkHashes[len(m.ChunkHashes)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipTypes(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTypes + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTypes + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTypes + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthTypes + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupTypes + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthTypes + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthTypes = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowTypes = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupTypes = fmt.Errorf("proto: unexpected end of group") +) diff --git a/snapshots/types/types.proto b/snapshots/types/types.proto new file mode 100644 index 000000000000..d39abe4e0b50 --- /dev/null +++ b/snapshots/types/types.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; +package cosmos_sdk.snapshots.v1; + +import "third_party/proto/gogoproto/gogo.proto"; + +option go_package = "github.com/cosmos/cosmos-sdk/snapshots/types"; + +// Snapshot contains snapshot info. +message Snapshot { + uint64 height = 1; + uint32 format = 2; + uint32 chunks = 3; + bytes hash = 4; + Metadata metadata = 5 [(gogoproto.nullable) = false]; +} + +// Metadata contains SDK-specific snapshot metadata. +message Metadata { + repeated bytes chunk_hashes = 1; // SHA-256 chunk hashes +} \ No newline at end of file diff --git a/snapshots/util.go b/snapshots/util.go new file mode 100644 index 000000000000..19bc87c1aab6 --- /dev/null +++ b/snapshots/util.go @@ -0,0 +1,162 @@ +package snapshots + +import ( + "errors" + "io" +) + +// ChunkWriter reads an input stream, splits it into fixed-size chunks, and writes them to a +// sequence of io.ReadClosers via a channel. +type ChunkWriter struct { + ch chan<- io.ReadCloser + pipe *io.PipeWriter + chunkSize uint64 + written uint64 + closed bool +} + +// NewChunkWriter creates a new ChunkWriter. If chunkSize is 0, no chunking will be done. +func NewChunkWriter(ch chan<- io.ReadCloser, chunkSize uint64) *ChunkWriter { + return &ChunkWriter{ + ch: ch, + chunkSize: chunkSize, + } +} + +// chunk creates a new chunk. +func (w *ChunkWriter) chunk() error { + if w.pipe != nil { + err := w.pipe.Close() + if err != nil { + return err + } + } + pr, pw := io.Pipe() + w.ch <- pr + w.pipe = pw + w.written = 0 + return nil +} + +// Close implements io.Closer. +func (w *ChunkWriter) Close() error { + if !w.closed { + w.closed = true + close(w.ch) + var err error + if w.pipe != nil { + err = w.pipe.Close() + } + return err + } + return nil +} + +// CloseWithError closes the writer and sends an error to the reader. +func (w *ChunkWriter) CloseWithError(err error) { + if !w.closed { + w.closed = true + close(w.ch) + if w.pipe != nil { + w.pipe.CloseWithError(err) + } + } +} + +// Write implements io.Writer. +func (w *ChunkWriter) Write(data []byte) (int, error) { + if w.closed { + return 0, errors.New("cannot write to closed ChunkWriter") + } + nTotal := 0 + for len(data) > 0 { + if w.pipe == nil || (w.written >= w.chunkSize && w.chunkSize > 0) { + err := w.chunk() + if err != nil { + return nTotal, err + } + } + + var writeSize uint64 + if w.chunkSize == 0 { + writeSize = uint64(len(data)) + } else { + writeSize = w.chunkSize - w.written + } + if writeSize > uint64(len(data)) { + writeSize = uint64(len(data)) + } + + n, err := w.pipe.Write(data[:writeSize]) + w.written += uint64(n) + nTotal += n + if err != nil { + return nTotal, err + } + data = data[writeSize:] + } + return nTotal, nil +} + +// ChunkReader reads chunks from a channel of io.ReadClosers and outputs them as an io.Reader +type ChunkReader struct { + ch <-chan io.ReadCloser + reader io.ReadCloser +} + +// NewChunkReader creates a new ChunkReader. +func NewChunkReader(ch <-chan io.ReadCloser) *ChunkReader { + return &ChunkReader{ch: ch} +} + +// next fetches the next chunk from the channel, or returns io.EOF if there are no more chunks. +func (r *ChunkReader) next() error { + reader, ok := <-r.ch + if !ok { + return io.EOF + } + r.reader = reader + return nil +} + +// Close implements io.ReadCloser. +func (r *ChunkReader) Close() error { + var err error + if r.reader != nil { + err = r.reader.Close() + r.reader = nil + } + for reader := range r.ch { + if e := reader.Close(); e != nil && err == nil { + err = e + } + } + return err +} + +// Read implements io.Reader. +func (r *ChunkReader) Read(p []byte) (int, error) { + if r.reader == nil { + err := r.next() + if err != nil { + return 0, err + } + } + n, err := r.reader.Read(p) + if err == io.EOF { + err = r.reader.Close() + r.reader = nil + if err != nil { + return 0, err + } + return r.Read(p) + } + return n, err +} + +// DrainChunks drains and closes all remaining chunks from a chunk channel. +func DrainChunks(chunks <-chan io.ReadCloser) { + for chunk := range chunks { + _ = chunk.Close() + } +} diff --git a/snapshots/util_test.go b/snapshots/util_test.go new file mode 100644 index 000000000000..72b236dafcac --- /dev/null +++ b/snapshots/util_test.go @@ -0,0 +1,166 @@ +package snapshots_test + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cosmos/cosmos-sdk/snapshots" +) + +func TestChunkWriter(t *testing.T) { + ch := make(chan io.ReadCloser, 100) + go func() { + chunkWriter := snapshots.NewChunkWriter(ch, 2) + + n, err := chunkWriter.Write([]byte{1, 2, 3}) + require.NoError(t, err) + assert.Equal(t, 3, n) + + n, err = chunkWriter.Write([]byte{4, 5, 6}) + require.NoError(t, err) + assert.Equal(t, 3, n) + + n, err = chunkWriter.Write([]byte{7, 8, 9}) + require.NoError(t, err) + assert.Equal(t, 3, n) + + err = chunkWriter.Close() + require.NoError(t, err) + + // closed writer should error + _, err = chunkWriter.Write([]byte{10}) + require.Error(t, err) + + // closing again should be fine + err = chunkWriter.Close() + require.NoError(t, err) + }() + + assert.Equal(t, [][]byte{{1, 2}, {3, 4}, {5, 6}, {7, 8}, {9}}, readChunks(ch)) + + // 0-sized chunks should return the whole body as one chunk + ch = make(chan io.ReadCloser, 100) + go func() { + chunkWriter := snapshots.NewChunkWriter(ch, 0) + _, err := chunkWriter.Write([]byte{1, 2, 3}) + require.NoError(t, err) + _, err = chunkWriter.Write([]byte{4, 5, 6}) + require.NoError(t, err) + err = chunkWriter.Close() + require.NoError(t, err) + }() + assert.Equal(t, [][]byte{{1, 2, 3, 4, 5, 6}}, readChunks(ch)) + + // closing with error should return the error + theErr := errors.New("boom") + ch = make(chan io.ReadCloser, 100) + go func() { + chunkWriter := snapshots.NewChunkWriter(ch, 2) + _, err := chunkWriter.Write([]byte{1, 2, 3}) + require.NoError(t, err) + chunkWriter.CloseWithError(theErr) + }() + chunk, err := ioutil.ReadAll(<-ch) + require.NoError(t, err) + assert.Equal(t, []byte{1, 2}, chunk) + _, err = ioutil.ReadAll(<-ch) + require.Error(t, err) + assert.Equal(t, theErr, err) + assert.Empty(t, ch) + + // closing immediately should return no chunks + ch = make(chan io.ReadCloser, 100) + chunkWriter := snapshots.NewChunkWriter(ch, 2) + err = chunkWriter.Close() + require.NoError(t, err) + assert.Empty(t, ch) +} + +func TestChunkReader(t *testing.T) { + + ch := makeChunks([][]byte{ + {1, 2, 3}, + {4}, + {}, + {5, 6}, + }) + chunkReader := snapshots.NewChunkReader(ch) + + buf := []byte{0, 0, 0, 0} + n, err := chunkReader.Read(buf) + require.NoError(t, err) + assert.Equal(t, 3, n) + assert.Equal(t, []byte{1, 2, 3, 0}, buf) + + buf = []byte{0, 0, 0, 0} + n, err = chunkReader.Read(buf) + require.NoError(t, err) + assert.Equal(t, 1, n) + assert.Equal(t, []byte{4, 0, 0, 0}, buf) + + buf = []byte{0, 0, 0, 0} + n, err = chunkReader.Read(buf) + require.NoError(t, err) + assert.Equal(t, 2, n) + assert.Equal(t, []byte{5, 6, 0, 0}, buf) + + buf = []byte{0, 0, 0, 0} + _, err = chunkReader.Read(buf) + require.Error(t, err) + assert.Equal(t, io.EOF, err) + + err = chunkReader.Close() + require.NoError(t, err) + + err = chunkReader.Close() // closing twice should be fine + require.NoError(t, err) + + // Empty channel should be fine + ch = makeChunks(nil) + chunkReader = snapshots.NewChunkReader(ch) + buf = make([]byte, 4) + _, err = chunkReader.Read(buf) + require.Error(t, err) + assert.Equal(t, io.EOF, err) + + // Using a pipe that closes with an error should return the error + theErr := errors.New("boom") + pr, pw := io.Pipe() + pch := make(chan io.ReadCloser, 1) + pch <- pr + pw.CloseWithError(theErr) + + chunkReader = snapshots.NewChunkReader(pch) + buf = make([]byte, 4) + _, err = chunkReader.Read(buf) + require.Error(t, err) + assert.Equal(t, theErr, err) + + // Closing the reader should close the writer + pr, pw = io.Pipe() + pch = make(chan io.ReadCloser, 2) + pch <- ioutil.NopCloser(bytes.NewBuffer([]byte{1, 2, 3})) + pch <- pr + close(pch) + + go func() { + chunkReader = snapshots.NewChunkReader(pch) + buf = []byte{0, 0, 0, 0} + _, err = chunkReader.Read(buf) + require.NoError(t, err) + assert.Equal(t, []byte{1, 2, 3, 0}, buf) + + err = chunkReader.Close() + require.NoError(t, err) + }() + + _, err = pw.Write([]byte{9, 9, 9}) + require.Error(t, err) + assert.Equal(t, err, io.ErrClosedPipe) +} diff --git a/store/iavl/store.go b/store/iavl/store.go index 3cc2a83b5ce1..be1929ce2196 100644 --- a/store/iavl/store.go +++ b/store/iavl/store.go @@ -233,6 +233,28 @@ func (st *Store) ReverseIterator(start, end []byte) types.Iterator { return newIAVLIterator(iTree, start, end, false) } +// Exports the IAVL store at the given version, returning an iavl.Exporter for the tree. +func (st *Store) Export(version int64) (*iavl.Exporter, error) { + istore, err := st.GetImmutable(version) + if err != nil { + return nil, fmt.Errorf("iavl export failed for version %v: %w", version, err) + } + tree, ok := istore.tree.(*immutableTree) + if !ok || tree == nil { + return nil, fmt.Errorf("iavl export failed: unable to fetch tree for version %v", version) + } + return tree.Export(), nil +} + +// Import imports an IAVL tree at the given version, returning an iavl.Importer for importing. +func (st *Store) Import(version int64) (*iavl.Importer, error) { + tree, ok := st.tree.(*iavl.MutableTree) + if !ok { + return nil, errors.New("iavl import failed: unable to find mutable tree") + } + return tree.Import(version) +} + // Handle gatest the latest height, if height is 0 func getHeight(tree Tree, req abci.RequestQuery) int64 { height := req.Height diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index bc59f62316ee..02e1c8984c09 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -1,16 +1,23 @@ package rootmulti import ( + "bufio" + "compress/zlib" "fmt" "io" + "math" + "sort" "strings" + protoio "github.com/gogo/protobuf/io" "github.com/pkg/errors" + tmiavl "github.com/tendermint/iavl" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto/tmhash" dbm "github.com/tendermint/tm-db" "github.com/cosmos/cosmos-sdk/codec" + "github.com/cosmos/cosmos-sdk/snapshots" "github.com/cosmos/cosmos-sdk/store/cachemulti" "github.com/cosmos/cosmos-sdk/store/dbadapter" "github.com/cosmos/cosmos-sdk/store/iavl" @@ -24,6 +31,11 @@ import ( const ( latestVersionKey = "s/latest" commitInfoKeyFmt = "s/%d" // s/ + + // Do not change chunk size without new snapshot format (must be uniform across nodes) + snapshotChunkSize = uint64(10e6) + snapshotBufferSize = int(snapshotChunkSize) + snapshotMaxItemSize = int(64e6) // FIXME SDK has no key/value limit, so we set an arbitrary limit ) var cdc = codec.New() @@ -485,6 +497,227 @@ func parsePath(path string) (storeName string, subpath string, err error) { return storeName, subpath, nil } +//---------------------- Snapshotting ------------------ + +// Snapshot implements snapshots.Snapshotter. The snapshot output for a given format must be +// identical across nodes such that chunks from different sources fit together. If the output for a +// given format changes (at the byte level), the snapshot format must be bumped - see +// TestMultistoreSnapshot_Checksum test. +func (rs *Store) Snapshot(height uint64, format uint32) (<-chan io.ReadCloser, error) { + if format != snapshots.CurrentFormat { + return nil, fmt.Errorf("%w %v", snapshots.ErrUnknownFormat, format) + } + if height == 0 { + return nil, errors.New("cannot snapshot height 0") + } + if height > uint64(rs.LastCommitID().Version) { + return nil, fmt.Errorf("cannot snapshot future height %v", height) + } + + // Collect stores to snapshot (only IAVL stores are supported) + type namedStore struct { + *iavl.Store + name string + } + stores := []namedStore{} + for key := range rs.stores { + switch store := rs.GetCommitKVStore(key).(type) { + case *iavl.Store: + stores = append(stores, namedStore{name: key.Name(), Store: store}) + case *transient.Store, *mem.Store: + // Non-persisted stores shouldn't be snapshotted + continue + default: + return nil, errors.Errorf("don't know how to snapshot store %q of type %T", key.Name(), store) + } + } + sort.Slice(stores, func(i, j int) bool { + return strings.Compare(stores[i].name, stores[j].name) == -1 + }) + + // Spawn goroutine to generate snapshot chunks and pass their io.ReadClosers through a channel + ch := make(chan io.ReadCloser) + go func() { + // Set up a stream pipeline to serialize snapshot nodes: + // ExportNode -> delimited Protobuf -> zlib -> buffer -> chunkWriter -> chan io.ReadCloser + chunkWriter := snapshots.NewChunkWriter(ch, snapshotChunkSize) + defer chunkWriter.Close() + bufWriter := bufio.NewWriterSize(chunkWriter, snapshotBufferSize) + defer func() { + if err := bufWriter.Flush(); err != nil { + chunkWriter.CloseWithError(err) + } + }() + zWriter, err := zlib.NewWriterLevel(bufWriter, 7) + if err != nil { + chunkWriter.CloseWithError(fmt.Errorf("zlib error: %w", err)) + return + } + defer func() { + if err := zWriter.Close(); err != nil { + chunkWriter.CloseWithError(err) + } + }() + protoWriter := protoio.NewDelimitedWriter(zWriter) + defer func() { + if err := protoWriter.Close(); err != nil { + chunkWriter.CloseWithError(err) + } + }() + + // Export each IAVL store. Stores are serialized as a stream of SnapshotItem Protobuf + // messages. The first item contains a SnapshotStore with store metadata (i.e. name), + // and the following messages contain a SnapshotNode (i.e. an ExportNode). Store changes + // are demarcated by new SnapshotStore items. + for _, store := range stores { + exporter, err := store.Export(int64(height)) + if err != nil { + chunkWriter.CloseWithError(err) + return + } + defer exporter.Close() + err = protoWriter.WriteMsg(&types.SnapshotItem{ + Item: &types.SnapshotItem_Store{ + Store: &types.SnapshotStoreItem{ + Name: store.name, + }, + }, + }) + if err != nil { + chunkWriter.CloseWithError(err) + return + } + + for { + node, err := exporter.Next() + if err == tmiavl.ExportDone { + break + } else if err != nil { + chunkWriter.CloseWithError(err) + return + } + err = protoWriter.WriteMsg(&types.SnapshotItem{ + Item: &types.SnapshotItem_IAVL{ + IAVL: &types.SnapshotIAVLItem{ + Key: node.Key, + Value: node.Value, + Height: int32(node.Height), + Version: node.Version, + }, + }, + }) + if err != nil { + chunkWriter.CloseWithError(err) + return + } + } + exporter.Close() + } + }() + + return ch, nil +} + +// Restore implements snapshots.Snapshotter. +func (rs *Store) Restore(height uint64, format uint32, chunks <-chan io.ReadCloser) error { + if format != snapshots.CurrentFormat { + return fmt.Errorf("%w %v", snapshots.ErrUnknownFormat, format) + } + if height == 0 { + return fmt.Errorf("%w: cannot restore snapshot at height 0", snapshots.ErrInvalidMetadata) + } + if height > math.MaxInt64 { + return fmt.Errorf("%w: snapshot height %v cannot exceed %v", snapshots.ErrInvalidMetadata, + height, math.MaxInt64) + } + + // Set up a restore stream pipeline + // chan io.ReadCloser -> chunkReader -> zlib -> delimited Protobuf -> ExportNode + chunkReader := snapshots.NewChunkReader(chunks) + defer chunkReader.Close() + zReader, err := zlib.NewReader(chunkReader) + if err != nil { + return fmt.Errorf("zlib error: %w", err) + } + defer zReader.Close() + protoReader := protoio.NewDelimitedReader(zReader, snapshotMaxItemSize) + defer protoReader.Close() + + // Import nodes into stores. The first item is expected to be a SnapshotItem containing + // a SnapshotStoreItem, telling us which store to import into. The following items will contain + // SnapshotNodeItem (i.e. ExportNode) until we reach the next SnapshotStoreItem or EOF. + var importer *tmiavl.Importer + for { + item := &types.SnapshotItem{} + err := protoReader.ReadMsg(item) + if err == io.EOF { + break + } else if err != nil { + return fmt.Errorf("invalid protobuf message: %w", err) + } + + switch item := item.Item.(type) { + case *types.SnapshotItem_Store: + if importer != nil { + err = importer.Commit() + if err != nil { + return fmt.Errorf("IAVL commit failed: %w", err) + } + importer.Close() + } + store, ok := rs.getStoreByName(item.Store.Name).(*iavl.Store) + if !ok || store == nil { + return fmt.Errorf("cannot import into non-IAVL store %q", item.Store.Name) + } + importer, err = store.Import(int64(height)) + if err != nil { + return fmt.Errorf("import failed: %w", err) + } + defer importer.Close() + + case *types.SnapshotItem_IAVL: + if importer == nil { + return fmt.Errorf("received IAVL node item before store item") + } + if item.IAVL.Height > math.MaxInt8 { + return fmt.Errorf("node height %v cannot exceed %v", item.IAVL.Height, math.MaxInt8) + } + node := &tmiavl.ExportNode{ + Key: item.IAVL.Key, + Value: item.IAVL.Value, + Height: int8(item.IAVL.Height), + Version: item.IAVL.Version, + } + // Protobuf does not differentiate between []byte{} as nil, but fortunately IAVL does + // not allow nil keys nor nil values for leaf nodes, so we can always set them to empty. + if node.Key == nil { + node.Key = []byte{} + } + if node.Height == 0 && node.Value == nil { + node.Value = []byte{} + } + err := importer.Add(node) + if err != nil { + return fmt.Errorf("IAVL node import failed: %w", err) + } + + default: + return fmt.Errorf("unknown snapshot item %T", item) + } + } + + if importer != nil { + err := importer.Commit() + if err != nil { + return fmt.Errorf("IAVL commit failed: %w", err) + } + importer.Close() + } + + flushCommitInfo(rs.db, int64(height), rs.buildCommitInfo(int64(height))) + return rs.LoadLatestVersion() +} + //---------------------------------------- // Note: why do we use key and params.key in different places. Seems like there should be only one key used. func (rs *Store) loadCommitStoreFromParams(key types.StoreKey, id types.CommitID, params storeParams) (types.CommitKVStore, error) { @@ -539,6 +772,25 @@ func (rs *Store) loadCommitStoreFromParams(key types.StoreKey, id types.CommitID } } +func (rs *Store) buildCommitInfo(version int64) commitInfo { + storeInfos := []storeInfo{} + for key, store := range rs.stores { + if store.GetStoreType() == types.StoreTypeTransient { + continue + } + storeInfos = append(storeInfos, storeInfo{ + Name: key.Name(), + Core: storeCore{ + CommitID: store.LastCommitID(), + }, + }) + } + return commitInfo{ + Version: version, + StoreInfos: storeInfos, + } +} + //---------------------------------------- // storeParams diff --git a/store/rootmulti/store_test.go b/store/rootmulti/store_test.go index a6f40dde3805..5577e7a66937 100644 --- a/store/rootmulti/store_test.go +++ b/store/rootmulti/store_test.go @@ -1,13 +1,22 @@ package rootmulti import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" "fmt" + "io" + "io/ioutil" + "math/rand" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" dbm "github.com/tendermint/tm-db" + "github.com/cosmos/cosmos-sdk/snapshots" "github.com/cosmos/cosmos-sdk/store/iavl" "github.com/cosmos/cosmos-sdk/store/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" @@ -442,6 +451,185 @@ func TestMultiStoreQuery(t *testing.T) { require.Equal(t, v2, qres.Value) } +func TestMultistoreSnapshot_Checksum(t *testing.T) { + // Chunks from different nodes must fit together, so all nodes must produce identical chunks. + // This checksum test makes sure that the byte stream remains identical. If the test fails + // without having changed the data (e.g. because the Protobuf or zlib encoding changes), + // snapshots.CurrentFormat must be bumped. + store := newMultiStoreWithGeneratedData(dbm.NewMemDB(), 5, 10000) + version := uint64(store.LastCommitID().Version) + + testcases := []struct { + format uint32 + chunkHashes []string + }{ + {1, []string{ + "503e5b51b657055b77e88169fadae543619368744ad15f1de0736c0a20482f24", + "e1a0daaa738eeb43e778aefd2805e3dd720798288a410b06da4b8459c4d8f72e", + "aa048b4ee0f484965d7b3b06822cf0772cdcaad02f3b1b9055e69f2cb365ef3c", + "7921eaa3ed4921341e504d9308a9877986a879fe216a099c86e8db66fcba4c63", + "a4a864e6c02c9fca5837ec80dc84f650b25276ed7e4820cf7516ced9f9901b86", + "ca2879ac6e7205d257440131ba7e72bef784cd61642e32b847729e543c1928b9", + }}, + } + for _, tc := range testcases { + tc := tc + t.Run(fmt.Sprintf("Format %v", tc.format), func(t *testing.T) { + chunks, err := store.Snapshot(version, tc.format) + require.NoError(t, err) + hashes := []string{} + for chunk := range chunks { + hasher := sha256.New() + _, err := io.Copy(hasher, chunk) + require.NoError(t, err) + hashes = append(hashes, hex.EncodeToString(hasher.Sum(nil))) + } + assert.Equal(t, tc.chunkHashes, hashes, + "Snapshot output for format %v has changed", tc.format) + }) + } +} + +func TestMultistoreSnapshot_Errors(t *testing.T) { + store := newMultiStoreWithMixedMountsAndBasicData(dbm.NewMemDB()) + + testcases := map[string]struct { + height uint64 + format uint32 + expectType error + }{ + "0 height": {0, snapshots.CurrentFormat, nil}, + "0 format": {1, 0, snapshots.ErrUnknownFormat}, + "unknown height": {9, snapshots.CurrentFormat, nil}, + "unknown format": {1, 9, snapshots.ErrUnknownFormat}, + } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + _, err := store.Snapshot(tc.height, tc.format) + require.Error(t, err) + if tc.expectType != nil { + assert.True(t, errors.Is(err, tc.expectType)) + } + }) + } +} + +func TestMultistoreRestore_Errors(t *testing.T) { + store := newMultiStoreWithMixedMounts(dbm.NewMemDB()) + + testcases := map[string]struct { + height uint64 + format uint32 + expectType error + }{ + "0 height": {0, snapshots.CurrentFormat, nil}, + "0 format": {1, 0, snapshots.ErrUnknownFormat}, + "unknown format": {1, 9, snapshots.ErrUnknownFormat}, + } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + err := store.Restore(tc.height, tc.format, nil) + require.Error(t, err) + if tc.expectType != nil { + assert.True(t, errors.Is(err, tc.expectType)) + } + }) + } +} + +func TestMultistoreSnapshotRestore(t *testing.T) { + source := newMultiStoreWithMixedMountsAndBasicData(dbm.NewMemDB()) + target := newMultiStoreWithMixedMounts(dbm.NewMemDB()) + version := uint64(source.LastCommitID().Version) + require.EqualValues(t, 3, version) + + chunks, err := source.Snapshot(version, snapshots.CurrentFormat) + require.NoError(t, err) + err = target.Restore(version, snapshots.CurrentFormat, chunks) + require.NoError(t, err) + + assert.Equal(t, source.LastCommitID(), target.LastCommitID()) + for key, sourceStore := range source.stores { + targetStore := target.getStoreByName(key.Name()).(types.CommitKVStore) + switch sourceStore.GetStoreType() { + case types.StoreTypeTransient: + assert.False(t, targetStore.Iterator(nil, nil).Valid(), + "transient store %v not empty", key.Name()) + default: + assertStoresEqual(t, sourceStore, targetStore, "store %q not equal", key.Name()) + } + } +} + +func BenchmarkMultistoreSnapshot100K(b *testing.B) { + benchmarkMultistoreSnapshot(b, 10, 10000) +} + +func BenchmarkMultistoreSnapshot1M(b *testing.B) { + benchmarkMultistoreSnapshot(b, 10, 100000) +} + +func BenchmarkMultistoreSnapshotRestore100K(b *testing.B) { + benchmarkMultistoreSnapshotRestore(b, 10, 10000) +} + +func BenchmarkMultistoreSnapshotRestore1M(b *testing.B) { + benchmarkMultistoreSnapshotRestore(b, 10, 100000) +} + +func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) { + b.StopTimer() + source := newMultiStoreWithGeneratedData(dbm.NewMemDB(), stores, storeKeys) + version := source.LastCommitID().Version + require.EqualValues(b, 1, version) + b.StartTimer() + + for i := 0; i < b.N; i++ { + target := NewStore(dbm.NewMemDB()) + for key := range source.stores { + target.MountStoreWithDB(key, types.StoreTypeIAVL, nil) + } + err := target.LoadLatestVersion() + require.NoError(b, err) + require.EqualValues(b, 0, target.LastCommitID().Version) + + chunks, err := source.Snapshot(uint64(version), snapshots.CurrentFormat) + require.NoError(b, err) + for reader := range chunks { + _, err := io.Copy(ioutil.Discard, reader) + require.NoError(b, err) + err = reader.Close() + require.NoError(b, err) + } + } +} + +func benchmarkMultistoreSnapshotRestore(b *testing.B, stores uint8, storeKeys uint64) { + b.StopTimer() + source := newMultiStoreWithGeneratedData(dbm.NewMemDB(), stores, storeKeys) + version := uint64(source.LastCommitID().Version) + require.EqualValues(b, 1, version) + b.StartTimer() + + for i := 0; i < b.N; i++ { + target := NewStore(dbm.NewMemDB()) + for key := range source.stores { + target.MountStoreWithDB(key, types.StoreTypeIAVL, nil) + } + err := target.LoadLatestVersion() + require.NoError(b, err) + require.EqualValues(b, 0, target.LastCommitID().Version) + + chunks, err := source.Snapshot(version, snapshots.CurrentFormat) + require.NoError(b, err) + err = target.Restore(version, snapshots.CurrentFormat, chunks) + require.NoError(b, err) + require.Equal(b, source.LastCommitID(), target.LastCommitID()) + } +} + //----------------------------------------------------------------------- // utils @@ -456,6 +644,76 @@ func newMultiStoreWithMounts(db dbm.DB, pruningOpts types.PruningOptions) *Store return store } +func newMultiStoreWithMixedMounts(db dbm.DB) *Store { + store := NewStore(db) + store.MountStoreWithDB(types.NewKVStoreKey("iavl1"), types.StoreTypeIAVL, nil) + store.MountStoreWithDB(types.NewKVStoreKey("iavl2"), types.StoreTypeIAVL, nil) + store.MountStoreWithDB(types.NewKVStoreKey("iavl3"), types.StoreTypeIAVL, nil) + store.MountStoreWithDB(types.NewTransientStoreKey("trans1"), types.StoreTypeTransient, nil) + store.LoadLatestVersion() + + return store +} + +func newMultiStoreWithMixedMountsAndBasicData(db dbm.DB) *Store { + store := newMultiStoreWithMixedMounts(db) + store1 := store.getStoreByName("iavl1").(types.CommitKVStore) + store2 := store.getStoreByName("iavl2").(types.CommitKVStore) + trans1 := store.getStoreByName("trans1").(types.KVStore) + + store1.Set([]byte("a"), []byte{1}) + store1.Set([]byte("b"), []byte{1}) + store2.Set([]byte("X"), []byte{255}) + store2.Set([]byte("A"), []byte{101}) + trans1.Set([]byte("x1"), []byte{91}) + store.Commit() + + store1.Set([]byte("b"), []byte{2}) + store1.Set([]byte("c"), []byte{3}) + store2.Set([]byte("B"), []byte{102}) + store.Commit() + + store1.Set([]byte{}, []byte{}) + store2.Set([]byte("C"), []byte{103}) + store2.Delete([]byte("X")) + trans1.Set([]byte("x2"), []byte{92}) + store.Commit() + + return store +} + +func newMultiStoreWithGeneratedData(db dbm.DB, stores uint8, storeKeys uint64) *Store { + multiStore := NewStore(db) + r := rand.New(rand.NewSource(49872768940)) // Fixed seed for deterministic tests + + keys := []*types.KVStoreKey{} + for i := uint8(0); i < stores; i++ { + key := types.NewKVStoreKey(fmt.Sprintf("store%v", i)) + multiStore.MountStoreWithDB(key, types.StoreTypeIAVL, nil) + keys = append(keys, key) + } + multiStore.LoadLatestVersion() + + for _, key := range keys { + store := multiStore.stores[key].(*iavl.Store) + for i := uint64(0); i < storeKeys; i++ { + k := make([]byte, 8) + v := make([]byte, 1024) + binary.BigEndian.PutUint64(k, i) + _, err := r.Read(v) + if err != nil { + panic(err) + } + store.Set(k, v) + } + } + + multiStore.Commit() + multiStore.LoadLatestVersion() + + return multiStore +} + func newMultiStoreWithModifiedMounts(db dbm.DB, pruningOpts types.PruningOptions) (*Store, *types.StoreUpgrades) { store := NewStore(db) store.pruningOpts = pruningOpts @@ -475,6 +733,25 @@ func newMultiStoreWithModifiedMounts(db dbm.DB, pruningOpts types.PruningOptions return store, upgrades } +func assertStoresEqual(t *testing.T, expect, actual types.CommitKVStore, msgAndArgs ...interface{}) { + assert.Equal(t, expect.LastCommitID(), actual.LastCommitID()) + expectIter := expect.Iterator(nil, nil) + expectMap := map[string][]byte{} + for ; expectIter.Valid(); expectIter.Next() { + expectMap[string(expectIter.Key())] = expectIter.Value() + } + require.NoError(t, expectIter.Error()) + + actualIter := expect.Iterator(nil, nil) + actualMap := map[string][]byte{} + for ; actualIter.Valid(); actualIter.Next() { + actualMap[string(actualIter.Key())] = actualIter.Value() + } + require.NoError(t, actualIter.Error()) + + assert.Equal(t, expectMap, actualMap, msgAndArgs...) +} + func checkStore(t *testing.T, store *Store, expect, got types.CommitID) { require.Equal(t, expect, got) require.Equal(t, expect, store.LastCommitID()) diff --git a/store/types/pruning.go b/store/types/pruning.go index c540443521ff..3fbd59fe57fa 100644 --- a/store/types/pruning.go +++ b/store/types/pruning.go @@ -28,6 +28,10 @@ var ( // will use when committing state, where keepEvery determines which committed // heights are flushed to disk and snapshotEvery determines which of these heights // are kept after pruning. +// +// SnapshotEvery also determines the state sync snapshot policy. Setting it to 0 or +// 1 disables state sync snapshots, while any other value will take state sync +// snapshots at that interval. type PruningOptions struct { KeepEvery int64 SnapshotEvery int64 diff --git a/store/types/store.go b/store/types/store.go index cb0d3dd5c2ce..8c39fc27dd39 100644 --- a/store/types/store.go +++ b/store/types/store.go @@ -4,6 +4,7 @@ import ( "fmt" "io" + "github.com/cosmos/cosmos-sdk/snapshots" abci "github.com/tendermint/tendermint/abci/types" tmkv "github.com/tendermint/tendermint/libs/kv" dbm "github.com/tendermint/tm-db" @@ -128,6 +129,7 @@ type CacheMultiStore interface { type CommitMultiStore interface { Committer MultiStore + snapshots.Snapshotter // Mount a store of type using the given db. // If db == nil, the new store will use the CommitMultiStore db. diff --git a/store/types/types.pb.go b/store/types/types.pb.go new file mode 100644 index 000000000000..2f484f9284b6 --- /dev/null +++ b/store/types/types.pb.go @@ -0,0 +1,951 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: store/types/types.proto + +package types + +import ( + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +// SnapshotItem is an item contained in a rootmulti.Store snapshot +type SnapshotItem struct { + // item gives the specific type of snapshot item + // + // Types that are valid to be assigned to Item: + // *SnapshotItem_Store + // *SnapshotItem_IAVL + Item isSnapshotItem_Item `protobuf_oneof:"item"` +} + +func (m *SnapshotItem) Reset() { *m = SnapshotItem{} } +func (m *SnapshotItem) String() string { return proto.CompactTextString(m) } +func (*SnapshotItem) ProtoMessage() {} +func (*SnapshotItem) Descriptor() ([]byte, []int) { + return fileDescriptor_8a5c5d9c27fabc9c, []int{0} +} +func (m *SnapshotItem) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SnapshotItem) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SnapshotItem.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SnapshotItem) XXX_Merge(src proto.Message) { + xxx_messageInfo_SnapshotItem.Merge(m, src) +} +func (m *SnapshotItem) XXX_Size() int { + return m.Size() +} +func (m *SnapshotItem) XXX_DiscardUnknown() { + xxx_messageInfo_SnapshotItem.DiscardUnknown(m) +} + +var xxx_messageInfo_SnapshotItem proto.InternalMessageInfo + +type isSnapshotItem_Item interface { + isSnapshotItem_Item() + MarshalTo([]byte) (int, error) + Size() int +} + +type SnapshotItem_Store struct { + Store *SnapshotStoreItem `protobuf:"bytes,1,opt,name=store,proto3,oneof" json:"store,omitempty"` +} +type SnapshotItem_IAVL struct { + IAVL *SnapshotIAVLItem `protobuf:"bytes,2,opt,name=iavl,proto3,oneof" json:"iavl,omitempty"` +} + +func (*SnapshotItem_Store) isSnapshotItem_Item() {} +func (*SnapshotItem_IAVL) isSnapshotItem_Item() {} + +func (m *SnapshotItem) GetItem() isSnapshotItem_Item { + if m != nil { + return m.Item + } + return nil +} + +func (m *SnapshotItem) GetStore() *SnapshotStoreItem { + if x, ok := m.GetItem().(*SnapshotItem_Store); ok { + return x.Store + } + return nil +} + +func (m *SnapshotItem) GetIAVL() *SnapshotIAVLItem { + if x, ok := m.GetItem().(*SnapshotItem_IAVL); ok { + return x.IAVL + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*SnapshotItem) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*SnapshotItem_Store)(nil), + (*SnapshotItem_IAVL)(nil), + } +} + +// SnapshotStore contains metadata about a snapshotted store +type SnapshotStoreItem struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +} + +func (m *SnapshotStoreItem) Reset() { *m = SnapshotStoreItem{} } +func (m *SnapshotStoreItem) String() string { return proto.CompactTextString(m) } +func (*SnapshotStoreItem) ProtoMessage() {} +func (*SnapshotStoreItem) Descriptor() ([]byte, []int) { + return fileDescriptor_8a5c5d9c27fabc9c, []int{1} +} +func (m *SnapshotStoreItem) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SnapshotStoreItem) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SnapshotStoreItem.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SnapshotStoreItem) XXX_Merge(src proto.Message) { + xxx_messageInfo_SnapshotStoreItem.Merge(m, src) +} +func (m *SnapshotStoreItem) XXX_Size() int { + return m.Size() +} +func (m *SnapshotStoreItem) XXX_DiscardUnknown() { + xxx_messageInfo_SnapshotStoreItem.DiscardUnknown(m) +} + +var xxx_messageInfo_SnapshotStoreItem proto.InternalMessageInfo + +func (m *SnapshotStoreItem) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +// SnapshotIAVLItem is an exported IAVL node +type SnapshotIAVLItem struct { + Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + Version int64 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` + Height int32 `protobuf:"varint,4,opt,name=height,proto3" json:"height,omitempty"` +} + +func (m *SnapshotIAVLItem) Reset() { *m = SnapshotIAVLItem{} } +func (m *SnapshotIAVLItem) String() string { return proto.CompactTextString(m) } +func (*SnapshotIAVLItem) ProtoMessage() {} +func (*SnapshotIAVLItem) Descriptor() ([]byte, []int) { + return fileDescriptor_8a5c5d9c27fabc9c, []int{2} +} +func (m *SnapshotIAVLItem) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SnapshotIAVLItem) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SnapshotIAVLItem.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SnapshotIAVLItem) XXX_Merge(src proto.Message) { + xxx_messageInfo_SnapshotIAVLItem.Merge(m, src) +} +func (m *SnapshotIAVLItem) XXX_Size() int { + return m.Size() +} +func (m *SnapshotIAVLItem) XXX_DiscardUnknown() { + xxx_messageInfo_SnapshotIAVLItem.DiscardUnknown(m) +} + +var xxx_messageInfo_SnapshotIAVLItem proto.InternalMessageInfo + +func (m *SnapshotIAVLItem) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *SnapshotIAVLItem) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +func (m *SnapshotIAVLItem) GetVersion() int64 { + if m != nil { + return m.Version + } + return 0 +} + +func (m *SnapshotIAVLItem) GetHeight() int32 { + if m != nil { + return m.Height + } + return 0 +} + +func init() { + proto.RegisterType((*SnapshotItem)(nil), "cosmos_sdk.store.v1.SnapshotItem") + proto.RegisterType((*SnapshotStoreItem)(nil), "cosmos_sdk.store.v1.SnapshotStoreItem") + proto.RegisterType((*SnapshotIAVLItem)(nil), "cosmos_sdk.store.v1.SnapshotIAVLItem") +} + +func init() { proto.RegisterFile("store/types/types.proto", fileDescriptor_8a5c5d9c27fabc9c) } + +var fileDescriptor_8a5c5d9c27fabc9c = []byte{ + // 325 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0x41, 0x4b, 0xfb, 0x30, + 0x18, 0xc6, 0x9b, 0xff, 0xba, 0xfd, 0xf5, 0x75, 0x87, 0x19, 0x45, 0x8b, 0x87, 0x38, 0x06, 0xce, + 0x5e, 0x4c, 0x51, 0xef, 0x82, 0x05, 0x61, 0x03, 0x4f, 0x19, 0x78, 0xf0, 0x32, 0xba, 0x2d, 0xb4, + 0x61, 0xeb, 0x52, 0x9a, 0xac, 0xb0, 0x6f, 0xe1, 0x17, 0xf0, 0xfb, 0x78, 0xdc, 0xd1, 0x93, 0x48, + 0xf7, 0x45, 0xa4, 0xc9, 0x86, 0xa2, 0xe2, 0xa5, 0x7d, 0x9e, 0xf0, 0xfc, 0x9e, 0xbc, 0xe1, 0x85, + 0x63, 0xa5, 0x65, 0xce, 0x03, 0xbd, 0xcc, 0xb8, 0xb2, 0x5f, 0x9a, 0xe5, 0x52, 0x4b, 0x7c, 0x30, + 0x96, 0x2a, 0x95, 0x6a, 0xa8, 0x26, 0x53, 0x6a, 0x32, 0xb4, 0xb8, 0x3c, 0xe9, 0xea, 0x44, 0xe4, + 0x93, 0x61, 0x16, 0xe5, 0x7a, 0x19, 0x98, 0x5c, 0x10, 0xcb, 0x58, 0x7e, 0x2a, 0x0b, 0x77, 0x9e, + 0x11, 0x34, 0x07, 0xf3, 0x28, 0x53, 0x89, 0xd4, 0x7d, 0xcd, 0x53, 0x7c, 0x03, 0x75, 0x53, 0xe2, + 0xa1, 0x36, 0xf2, 0xf7, 0xae, 0xba, 0xf4, 0x97, 0x76, 0xba, 0x25, 0x06, 0xd5, 0x41, 0x85, 0xf5, + 0x1c, 0x66, 0x31, 0x7c, 0x07, 0xae, 0x88, 0x8a, 0x99, 0xf7, 0xcf, 0xe0, 0x67, 0x7f, 0xe2, 0xfd, + 0xdb, 0x87, 0xfb, 0x8a, 0x0e, 0x77, 0xca, 0xb7, 0x53, 0xb7, 0x72, 0x3d, 0x87, 0x19, 0x3c, 0x6c, + 0x80, 0x2b, 0x34, 0x4f, 0x3b, 0xe7, 0xb0, 0xff, 0xe3, 0x32, 0x8c, 0xc1, 0x9d, 0x47, 0xa9, 0x1d, + 0x71, 0x97, 0x19, 0xdd, 0x99, 0x41, 0xeb, 0x7b, 0x2d, 0x6e, 0x41, 0x6d, 0xca, 0x97, 0x26, 0xd6, + 0x64, 0x95, 0xc4, 0x87, 0x50, 0x2f, 0xa2, 0xd9, 0x82, 0x9b, 0xf1, 0x9a, 0xcc, 0x1a, 0xec, 0xc1, + 0xff, 0x82, 0xe7, 0x4a, 0xc8, 0xb9, 0x57, 0x6b, 0x23, 0xbf, 0xc6, 0xb6, 0x16, 0x1f, 0x41, 0x23, + 0xe1, 0x22, 0x4e, 0xb4, 0xe7, 0xb6, 0x91, 0x5f, 0x67, 0x1b, 0x17, 0x86, 0x2f, 0x25, 0x41, 0xab, + 0x92, 0xa0, 0xf7, 0x92, 0xa0, 0xa7, 0x35, 0x71, 0x56, 0x6b, 0xe2, 0xbc, 0xae, 0x89, 0xf3, 0xe8, + 0xc7, 0x42, 0x27, 0x8b, 0x11, 0x1d, 0xcb, 0x34, 0xb0, 0x6f, 0xdf, 0xfc, 0x2e, 0xd4, 0x64, 0x1a, + 0x7c, 0xd9, 0xe1, 0xa8, 0x61, 0x36, 0x70, 0xfd, 0x11, 0x00, 0x00, 0xff, 0xff, 0x11, 0x49, 0x37, + 0xf2, 0xd9, 0x01, 0x00, 0x00, +} + +func (m *SnapshotItem) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SnapshotItem) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SnapshotItem) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Item != nil { + { + size := m.Item.Size() + i -= size + if _, err := m.Item.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + } + } + return len(dAtA) - i, nil +} + +func (m *SnapshotItem_Store) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SnapshotItem_Store) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Store != nil { + { + size, err := m.Store.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} +func (m *SnapshotItem_IAVL) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SnapshotItem_IAVL) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.IAVL != nil { + { + size, err := m.IAVL.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + return len(dAtA) - i, nil +} +func (m *SnapshotStoreItem) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SnapshotStoreItem) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SnapshotStoreItem) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintTypes(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SnapshotIAVLItem) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SnapshotIAVLItem) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SnapshotIAVLItem) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Height != 0 { + i = encodeVarintTypes(dAtA, i, uint64(m.Height)) + i-- + dAtA[i] = 0x20 + } + if m.Version != 0 { + i = encodeVarintTypes(dAtA, i, uint64(m.Version)) + i-- + dAtA[i] = 0x18 + } + if len(m.Value) > 0 { + i -= len(m.Value) + copy(dAtA[i:], m.Value) + i = encodeVarintTypes(dAtA, i, uint64(len(m.Value))) + i-- + dAtA[i] = 0x12 + } + if len(m.Key) > 0 { + i -= len(m.Key) + copy(dAtA[i:], m.Key) + i = encodeVarintTypes(dAtA, i, uint64(len(m.Key))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func encodeVarintTypes(dAtA []byte, offset int, v uint64) int { + offset -= sovTypes(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *SnapshotItem) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Item != nil { + n += m.Item.Size() + } + return n +} + +func (m *SnapshotItem_Store) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Store != nil { + l = m.Store.Size() + n += 1 + l + sovTypes(uint64(l)) + } + return n +} +func (m *SnapshotItem_IAVL) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.IAVL != nil { + l = m.IAVL.Size() + n += 1 + l + sovTypes(uint64(l)) + } + return n +} +func (m *SnapshotStoreItem) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + return n +} + +func (m *SnapshotIAVLItem) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Key) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + l = len(m.Value) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + if m.Version != 0 { + n += 1 + sovTypes(uint64(m.Version)) + } + if m.Height != 0 { + n += 1 + sovTypes(uint64(m.Height)) + } + return n +} + +func sovTypes(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozTypes(x uint64) (n int) { + return sovTypes(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *SnapshotItem) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SnapshotItem: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SnapshotItem: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Store", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &SnapshotStoreItem{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Item = &SnapshotItem_Store{v} + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field IAVL", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &SnapshotIAVLItem{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Item = &SnapshotItem_IAVL{v} + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SnapshotStoreItem) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SnapshotStoreItem: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SnapshotStoreItem: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SnapshotIAVLItem) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SnapshotIAVLItem: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SnapshotIAVLItem: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) + if m.Key == nil { + m.Key = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Value = append(m.Value[:0], dAtA[iNdEx:postIndex]...) + if m.Value == nil { + m.Value = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) + } + m.Version = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Version |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Height", wireType) + } + m.Height = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Height |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipTypes(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTypes + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTypes + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTypes + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthTypes + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupTypes + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthTypes + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthTypes = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowTypes = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupTypes = fmt.Errorf("proto: unexpected end of group") +) diff --git a/store/types/types.proto b/store/types/types.proto new file mode 100644 index 000000000000..e39c467dd918 --- /dev/null +++ b/store/types/types.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; +package cosmos_sdk.store.v1; + +import "third_party/proto/gogoproto/gogo.proto"; + +option go_package = "github.com/cosmos/cosmos-sdk/store/types"; + +// SnapshotItem is an item contained in a rootmulti.Store snapshot +message SnapshotItem { + // item gives the specific type of snapshot item + oneof item { + SnapshotStoreItem store = 1; + SnapshotIAVLItem iavl = 2 [(gogoproto.customname) = "IAVL"]; + } +} + +// SnapshotStore contains metadata about a snapshotted store +message SnapshotStoreItem { + string name = 1; +} + +// SnapshotIAVLItem is an exported IAVL node +message SnapshotIAVLItem { + bytes key = 1; + bytes value = 2; + int64 version = 3; + int32 height = 4; +} \ No newline at end of file diff --git a/types/events.go b/types/events.go index 10723931e9cd..ae542afd2696 100644 --- a/types/events.go +++ b/types/events.go @@ -65,7 +65,7 @@ func NewEvent(ty string, attrs ...Attribute) Event { e := Event{Type: ty} for _, attr := range attrs { - e.Attributes = append(e.Attributes, attr.ToKVPair()) + e.Attributes = append(e.Attributes, attr.ToEventAttributes()) } return e @@ -90,10 +90,15 @@ func (a Attribute) ToKVPair() tmkv.Pair { return tmkv.Pair{Key: toBytes(a.Key), Value: toBytes(a.Value)} } +// ToEventAttributes converts an Attribute object into a Tendermint EventAttributes object. +func (a Attribute) ToEventAttributes() abci.EventAttribute { + return abci.EventAttribute{Key: toBytes(a.Key), Value: toBytes(a.Value)} +} + // AppendAttributes adds one or more attributes to an Event. func (e Event) AppendAttributes(attrs ...Attribute) Event { for _, attr := range attrs { - e.Attributes = append(e.Attributes, attr.ToKVPair()) + e.Attributes = append(e.Attributes, attr.ToEventAttributes()) } return e } diff --git a/x/bank/keeper/keeper_test.go b/x/bank/keeper/keeper_test.go index 2ae3f8131d7f..3a1dcdb9e453 100644 --- a/x/bank/keeper/keeper_test.go +++ b/x/bank/keeper/keeper_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/suite" abci "github.com/tendermint/tendermint/abci/types" - tmkv "github.com/tendermint/tendermint/libs/kv" tmtime "github.com/tendermint/tendermint/types/time" "github.com/cosmos/cosmos-sdk/simapp" @@ -458,23 +457,23 @@ func (suite *IntegrationTestSuite) TestMsgSendEvents() { event1 := sdk.Event{ Type: types.EventTypeTransfer, - Attributes: []tmkv.Pair{}, + Attributes: []abci.EventAttribute{}, } event1.Attributes = append( event1.Attributes, - tmkv.Pair{Key: []byte(types.AttributeKeyRecipient), Value: []byte(addr2.String())}, + abci.EventAttribute{Key: []byte(types.AttributeKeyRecipient), Value: []byte(addr2.String())}, ) event1.Attributes = append( event1.Attributes, - tmkv.Pair{Key: []byte(sdk.AttributeKeyAmount), Value: []byte(newCoins.String())}, + abci.EventAttribute{Key: []byte(sdk.AttributeKeyAmount), Value: []byte(newCoins.String())}, ) event2 := sdk.Event{ Type: sdk.EventTypeMessage, - Attributes: []tmkv.Pair{}, + Attributes: []abci.EventAttribute{}, } event2.Attributes = append( event2.Attributes, - tmkv.Pair{Key: []byte(types.AttributeKeySender), Value: []byte(addr.String())}, + abci.EventAttribute{Key: []byte(types.AttributeKeySender), Value: []byte(addr.String())}, ) suite.Require().Equal(abci.Event(event1), events[0]) @@ -532,11 +531,11 @@ func (suite *IntegrationTestSuite) TestMsgMultiSendEvents() { event1 := sdk.Event{ Type: sdk.EventTypeMessage, - Attributes: []tmkv.Pair{}, + Attributes: []abci.EventAttribute{}, } event1.Attributes = append( event1.Attributes, - tmkv.Pair{Key: []byte(types.AttributeKeySender), Value: []byte(addr.String())}, + abci.EventAttribute{Key: []byte(types.AttributeKeySender), Value: []byte(addr.String())}, ) suite.Require().Equal(abci.Event(event1), events[0]) @@ -554,34 +553,34 @@ func (suite *IntegrationTestSuite) TestMsgMultiSendEvents() { event2 := sdk.Event{ Type: sdk.EventTypeMessage, - Attributes: []tmkv.Pair{}, + Attributes: []abci.EventAttribute{}, } event2.Attributes = append( event2.Attributes, - tmkv.Pair{Key: []byte(types.AttributeKeySender), Value: []byte(addr2.String())}, + abci.EventAttribute{Key: []byte(types.AttributeKeySender), Value: []byte(addr2.String())}, ) event3 := sdk.Event{ Type: types.EventTypeTransfer, - Attributes: []tmkv.Pair{}, + Attributes: []abci.EventAttribute{}, } event3.Attributes = append( event3.Attributes, - tmkv.Pair{Key: []byte(types.AttributeKeyRecipient), Value: []byte(addr3.String())}, + abci.EventAttribute{Key: []byte(types.AttributeKeyRecipient), Value: []byte(addr3.String())}, ) event3.Attributes = append( event3.Attributes, - tmkv.Pair{Key: []byte(sdk.AttributeKeyAmount), Value: []byte(newCoins.String())}) + abci.EventAttribute{Key: []byte(sdk.AttributeKeyAmount), Value: []byte(newCoins.String())}) event4 := sdk.Event{ Type: types.EventTypeTransfer, - Attributes: []tmkv.Pair{}, + Attributes: []abci.EventAttribute{}, } event4.Attributes = append( event4.Attributes, - tmkv.Pair{Key: []byte(types.AttributeKeyRecipient), Value: []byte(addr4.String())}, + abci.EventAttribute{Key: []byte(types.AttributeKeyRecipient), Value: []byte(addr4.String())}, ) event4.Attributes = append( event4.Attributes, - tmkv.Pair{Key: []byte(sdk.AttributeKeyAmount), Value: []byte(newCoins2.String())}, + abci.EventAttribute{Key: []byte(sdk.AttributeKeyAmount), Value: []byte(newCoins2.String())}, ) suite.Require().Equal(abci.Event(event1), events[1])