From 3086c7c136451bcf3f49136ca2c0d0360a0b387e Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Thu, 6 Oct 2022 18:27:05 -0500 Subject: [PATCH 01/20] update proposal --- docs/architecture/adr-038-state-listening.md | 384 +++++++++++++++++-- 1 file changed, 343 insertions(+), 41 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index 88fca031551..2a9e62808e2 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -206,16 +206,16 @@ func (rs *Store) CacheMultiStore() types.CacheMultiStore { #### Streaming service -We will introduce a new `StreamingService` interface for exposing `WriteListener` data streams to external consumers. -In addition to streaming state changes as `StoreKVPair`s, the interface satisfies an `ABCIListener` interface that plugs +We will introduce a new `StreamingService` struct for exposing `WriteListener` data streams to external consumers. +In addition to streaming state changes as `StoreKVPair`s, the strut satisfies an `ABCIListener` interface that plugs into the BaseApp and relays ABCI requests and responses so that the service can group the state changes with the ABCI requests that affected them and the ABCI responses they affected. The `ABCIListener` interface also exposes a -`ListenSuccess` method which is (optionally) used by the `BaseApp` to await positive acknowledgement of message -receipt from the `StreamingService`. +`ListenKVStorePair` method which is used by the `StoreKVPairWriteListener` to stream state change events. ```go // ABCIListener interface used to hook into the ABCI message processing of the BaseApp type ABCIListener interface { +<<<<<<< Updated upstream // ListenBeginBlock updates the streaming service with the latest BeginBlock messages ListenBeginBlock(ctx types.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error // ListenEndBlock updates the steaming service with the latest EndBlock messages @@ -241,6 +241,131 @@ type StreamingService interface { ABCIListener // Closer interface io.Closer +======= + // ListenBeginBlock updates the streaming service with the latest BeginBlock messages + ListenBeginBlock(blockHeight int64, req []byte, res []byte) error + // ListenEndBlock updates the steaming service with the latest EndBlock messages + ListenEndBlock(blockHeight int64, req []byte, res []byte) error + // ListenDeliverTx updates the steaming service with the latest DeliverTx messages + ListenDeliverTx(blockHeight int64, req []byte, res []byte) error + // ListenStoreKVPair updates the steaming service with the latest StoreKVPair messages + ListenStoreKVPair(blockHeight int64, data []byte) error +} + +// StreamingService struct for registering WriteListeners with the BaseApp and updating the service with the ABCI messages using the hooks +type StreamingService struct { + // Listeners returns the streaming service's listeners for the BaseApp to register + Listeners() map[types.StoreKey][]store.WriteListener + // ABCIListener interface for hooking into the ABCI messages from inside the BaseApp + ABCIListener ABCIListener + // StopNodeOnErr stops the node when true + StopNodeOnErr bool +} +``` + +We will also introduce a new `KVStoreListener` struct as the writer for use in `StoreKVPairWriteListener`. `KVStoreListener` will expose `ABCIListener` for streaming state change events via `ABCIListener.ListenStoreKVPair`, a `BlockHeight` func for determining the block height for which the state change events belong to and a boolean flag for determine whether the node needs to be stopped or not. + +```go +// KVStoreListener is used so that we do not need to update the underlying +// io.Writer inside the StoreKVPairWriteListener everytime we begin writing +type KVStoreListener struct { + BlockHeight func() int64 + listener ABCIListener + stopNodeOnErr bool +} + +// NewKVStoreListener create an instance of an NewKVStoreListener that sends StoreKVPair data to listening service +func NewKVStoreListener( + listener ABCIListener, + stopNodeOnErr bool, + blockHeight func() int64, +) *KVStoreListener { + return &KVStoreListener{ + listener: listener, + stopNodeOnErr: stopNodeOnErr, + BlockHeight: blockHeight, + } +} + +// Write satisfies io.Writer +func (iw *KVStoreListener) Write(b []byte) (int, error) { + blockHeight := iw.BlockHeight() + if err := iw.listener.ListenStoreKVPair(blockHeight, b); err != nil { + if iw.stopNodeOnErr { + panic(err) + } + return 0, err + } + return len(b), nil +} +``` + +We will expose a `RegisterStreamingService` function for the App to register streaming services. +```go +// RegisterStreamingService registers the ABCI streaming service provided by the streaming plugin. +func RegisterStreamingService( + bApp *BaseApp, + appOpts servertypes.AppOptions, + kodec codec.BinaryCodec, + keys map[string]*types.KVStoreKey, + streamingService interface{}, +) error { + // type checking + abciListener, ok := streamingService.(ABCIListener) + if !ok { + return fmt.Errorf("failed to register streaming service: failed type check %v", streamingService) + } + + // expose keys + keysKey := fmt.Sprintf("%s.%s", StreamingTomlKey, StreamingKeysTomlKey) + exposeKeysStr := cast.ToStringSlice(appOpts.Get(keysKey)) + exposeStoreKeys := exposeStoreKeys(exposeKeysStr, keys) + stopNodeOnErrKey := fmt.Sprintf("%s.%s", StreamingTomlKey, StreamingStopNodeOnErrTomlKey) + stopNodeOnErr := cast.ToBool(appOpts.Get(stopNodeOnErrKey)) + blockHeightFn := func() int64 { return bApp.deliverState.ctx.BlockHeight() } + writer := NewKVStoreListener(abciListener, stopNodeOnErr, blockHeightFn) + listener := types.NewStoreKVPairWriteListener(writer, kodec) + listeners := make(map[types.StoreKey][]types.WriteListener, len(exposeStoreKeys)) + for _, key := range exposeStoreKeys { + listeners[key] = append(listeners[key], listener) + } + + bApp.SetStreamingService(StreamingService{ + Listeners: listeners, + ABCIListener: abciListener, + StopNodeOnErr: stopNodeOnErr, + }) + + return nil +} + +func exposeAll(list []string) bool { + for _, ele := range list { + if ele == "*" { + return true + } + } + return false +} + +func exposeStoreKeys(keysStr []string, keys map[string]*types.KVStoreKey) []types.StoreKey { + var exposeStoreKeys []types.StoreKey + if exposeAll(keysStr) { + exposeStoreKeys = make([]types.StoreKey, 0, len(keys)) + for _, storeKey := range keys { + exposeStoreKeys = append(exposeStoreKeys, storeKey) + } + } else { + exposeStoreKeys = make([]types.StoreKey, 0, len(keysStr)) + for _, keyStr := range keysStr { + if storeKey, ok := keys[keyStr]; ok { + exposeStoreKeys = append(exposeStoreKeys, storeKey) + } + } + } + + return exposeStoreKeys +>>>>>>> Stashed changes } ``` @@ -252,12 +377,17 @@ We will add a new method to the `BaseApp` to enable the registration of `Streami // SetStreamingService is used to set a streaming service into the BaseApp hooks and load the listeners into the multistore func (app *BaseApp) SetStreamingService(s StreamingService) { // add the listeners for each StoreKey - for key, lis := range s.Listeners() { + for key, lis := range s.Listeners { app.cms.AddListeners(key, lis) } // register the StreamingService within the BaseApp // BaseApp will pass BeginBlock, DeliverTx, and EndBlock requests and responses to the streaming services to update their ABCI context +<<<<<<< Updated upstream app.abciListeners = append(app.abciListeners, s) +======= + app.abciListener = s.ABCIListener + app.stopNodeOnStreamingErr = s.StopNodeOnErr +>>>>>>> Stashed changes } ``` @@ -269,6 +399,7 @@ func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeg ... +<<<<<<< Updated upstream // call the hooks with the BeginBlock messages wg := new(sync.WaitGroup) for _, streamingListener := range app.abciListeners { @@ -291,13 +422,30 @@ func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeg app.logger.Error("BeginBlock listening hook failed", "height", req.Header.Height, "err", err) } }() +======= + // call the streaming service hook with the BeginBlock messages + if app.abciListener != nil { + reqBz, err := req.Marshal() + if err != nil { + panic(err) + } + resBz, err := res.Marshal() + if err != nil { + panic(err) + } + blockHeight := app.deliverState.ctx.BlockHeight() + if err := app.abciListener.ListenBeginBlock(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("BeginBlock listening hook failed", "height", req.Header.Height, "err", err) + if app.stopNodeOnStreamingErr { + panic(err) + } +>>>>>>> Stashed changes } } // wait for all the listener calls to finish wg.Wait() return res -} ``` ```go @@ -305,6 +453,7 @@ func (app *BaseApp) EndBlock(req abci.RequestEndBlock) (res abci.ResponseEndBloc ... +<<<<<<< Updated upstream // Call the streaming service hooks with the EndBlock messages wg := new(sync.WaitGroup) for _, streamingListener := range app.abciListeners { @@ -331,11 +480,61 @@ func (app *BaseApp) EndBlock(req abci.RequestEndBlock) (res abci.ResponseEndBloc } // wait for all the listener calls to finish wg.Wait() +======= + // call the streaming service hook with the EndBlock messages + if app.abciListener != nil { + reqBz, err := req.Marshal() + if err != nil { + panic(err) + } + resBz, err := res.Marshal() + if err != nil { + panic(err) + } + blockHeight := app.deliverState.ctx.BlockHeight() + if err := app.abciListener.ListenEndBlock(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("EndBlock listening hook failed", "height", req.Height, "err", err) + if app.stopNodeOnStreamingErr { + panic(err) + } + } + } return res -} ``` +```go +func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx { + + var abciRes abci.ResponseDeliverTx + defer func() { + // call the streaming service hook with the EndBlock messages + if app.abciListener != nil { + reqBz, err := req.Marshal() + if err != nil { + panic(err) + } + resBz, err := abciRes.Marshal() + if err != nil { + panic(err) + } + blockHeight := app.deliverState.ctx.BlockHeight() + if err := app.abciListener.ListenDeliverTx(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("DeliverTx listening hook failed", "err", err) + if app.stopNodeOnStreamingErr { + panic(err) + } + } + } + }() + + ... +>>>>>>> Stashed changes + + return res +``` + +<<<<<<< Updated upstream ```go func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx { @@ -372,44 +571,82 @@ func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx ... return res -} -``` - -#### Plugin system +======= +#### Go plugin system -We propose a plugin architecture to load and run `StreamingService` implementations. We will introduce a plugin -loading/preloading system that is used to load, initialize, inject, run, and stop Cosmos-SDK plugins. Each plugin -must implement the following interface: +We propose a plugin architecture to load and run `StreamingService` and other types of implementations. We will introduce a plugin +system over gRPC that is used to load and run Cosmos-SDK plugins. The plugin system uses [hashicorp/go-plugin](https://github.com/hashicorp/go-plugin). +Each plugin must have a struct that implements the `plugin.Plugin` interface and an `Impl` interface for processing messages over gRPC. +Each plugin must also have a message protocol defined for the gRPC service: ```go -// Plugin is the base interface for all kinds of cosmos-sdk plugins +// ListenerPlugin is the base struc for all kinds of go-plugin implementations // It will be included in interfaces of different Plugins -type Plugin interface { - // Name should return unique name of the plugin - Name() string - - // Version returns current version of the plugin - Version() string +type ListenerPlugin interface { + // GRPCPlugin must still implement the Plugin interface + plugin.Plugin + // Concrete implementation, written in Go. This is only used for plugins + // that are written in Go. + Impl baseapp.ABCIListener +} - // Init is called once when the Plugin is being loaded - // The plugin is passed the AppOptions for configuration - // A plugin will not necessarily have a functional Init - Init(env serverTypes.AppOptions) error +func (p *ListenerGRPCPlugin) GRPCServer(_ *plugin.GRPCBroker, s *grpc.Server) error { + RegisterABCIListenerServiceServer(s, &GRPCServer{Impl: p.Impl}) + return nil +} - // Closer interface for shutting down the plugin process - io.Closer +func (p *ListenerGRPCPlugin) GRPCClient( + _ context.Context, + _ *plugin.GRPCBroker, + c *grpc.ClientConn, +) (interface{}, error) { + return &GRPCClient{client: NewABCIListenerServiceClient(c)}, nil +>>>>>>> Stashed changes } ``` -The `Name` method returns a plugin's name. -The `Version` method returns a plugin's version. -The `Init` method initializes a plugin with the provided `AppOptions`. -The io.Closer is used to shut down the plugin service. +The `plugin.Plugin` interface has two methods `Client` and `Server`. For our GRPC service these are `GRPCClient` and `GRPCServer` +The `Impl` field holds the concrete implementation of our `baseapp.ABCIListener` interface written in Go. +Note: this is only used for plugins that are written in Go. + +The `StreamingService` protocol: + +```protobuf +syntax = "proto3"; + +package cosmos.sdk.grpc.abci.v1; + +option go_package = "github.com/cosmos/cosmos-sdk/streaming/plugins/abci/grpc_abci_v1"; +option java_multiple_files = true; +option java_outer_classname = "AbciListenerProto"; +option java_package = "network.cosmos.sdk.grpc.abci.v1"; + +// PutRequest is used for storing ABCI request and response +// and Store KV data for streaming to external grpc service. +message PutRequest { + int64 block_height = 1; + bytes req = 2; + bytes res = 3; + bytes store_kv_pair = 4; + int64 store_kv_pair_idx = 5; + int64 tx_idx = 6; +} + +message Empty {} + +service ABCIListenerService { + rpc ListenBeginBlock(PutRequest) returns (Empty); + rpc ListenEndBlock(PutRequest) returns (Empty); + rpc ListenDeliverTx(PutRequest) returns (Empty); + rpc ListenStoreKVPair(PutRequest) returns (Empty); +} +``` -For the purposes of this ADR we introduce a single kind of plugin- a state streaming plugin. -We will define a `StateStreamingPlugin` interface which extends the above `Plugin` interface to support a state streaming service. +For the purposes of this ADR we introduce a single state streaming plugin loading system. This loading system returns `(interface{}, error)`. +This provides the advantage of using versioned plugins where the plugin interface and gRPC protocol change over time. ```go +<<<<<<< Updated upstream // StateStreamingPlugin interface for plugins that load a baseapp.StreamingService onto a baseapp.BaseApp type StateStreamingPlugin interface { // Register configures and registers the plugin streaming service with the BaseApp @@ -417,14 +654,39 @@ type StateStreamingPlugin interface { // Start starts the background streaming process of the plugin streaming service Start(wg *sync.WaitGroup) error +======= +func NewStreamingPlugin(name string, logLevel string) (interface{}, error) { + logger := hclog.New(&hclog.LoggerOptions{ + Output: hclog.DefaultOutput, + Level: toHclogLevel(logLevel), + Name: fmt.Sprintf("plugin.%s", name), + }) + + // We're a host. Start by launching the streaming process. + env := os.Getenv(GetPluginEnvKey(name)) + client := plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: HandshakeMap[name], + Plugins: PluginMap, + Cmd: exec.Command("sh", "-c", env), + Logger: logger, + AllowedProtocols: []plugin.Protocol{ + plugin.ProtocolNetRPC, plugin.ProtocolGRPC}, + }) + + // Connect via RPC + rpcClient, err := client.Client() + if err != nil { + return nil, err + } +>>>>>>> Stashed changes - // Plugin is the base Plugin interface - Plugin + // Request streaming plugin + return rpcClient.Dispense(name) } + ``` -The `Register` method is used during App construction to register the plugin's streaming service with an App's BaseApp using the BaseApp's `SetStreamingService` method. -The `Start` method is used during App construction to start the registered plugin streaming services and maintain synchronization with them. +The `RegisterStreamingService` function is used during App construction to register a plugin with the App's BaseApp using the `NewStreamingPlugin` method. e.g. in `NewSimApp`: @@ -447,6 +709,7 @@ func NewSimApp( evidencetypes.StoreKey, ibctransfertypes.StoreKey, capabilitytypes.StoreKey, ) +<<<<<<< Updated upstream pluginsOnKey := fmt.Sprintf("%s.%s", plugin.PLUGINS_TOML_KEY, plugin.PLUGINS_ON_TOML_KEY) if cast.ToBool(appOpts.Get(pluginsOnKey)) { // this loads the preloaded and any plugins found in `plugins.dir` @@ -472,18 +735,34 @@ func NewSimApp( } } +======= +>>>>>>> Stashed changes ... + // enable streaming + enableKey := fmt.Sprintf("%s.%s", baseapp.StreamingTomlKey, baseapp.StreamingEnableTomlKey) + if enable := cast.ToBool(appOpts.Get(enableKey)); enable { + pluginKey := fmt.Sprintf("%s.%s", baseapp.StreamingTomlKey, baseapp.StreamingPluginTomlKey) + pluginName := cast.ToString(appOpts.Get(pluginKey)) + logLevel := cast.ToString(appOpts.Get(flags.FlagLogLevel)) + plugin, err := streaming.NewStreamingPlugin(pluginName, logLevel) + if err != nil { + tmos.Exit(err.Error()) + } + if err := baseapp.RegisterStreamingService(bApp, appOpts, appCodec, keys, plugin); err != nil { + tmos.Exit(err.Error()) + } + } + return app -} ``` - #### Configuration -The plugin system will be configured within an app's app.toml file. +The plugin system will be configured within an App's TOML configuration files. ```toml +<<<<<<< Updated upstream [plugins] on = false # turn the plugin system, as a whole, on or off enabled = ["list", "of", "plugin", "names", "to", "enable"] @@ -493,9 +772,19 @@ The plugin system will be configured within an app's app.toml file. There will be three parameters for configuring the plugin system: `plugins.on`, `plugins.enabled` and `plugins.dir`. `plugins.on` is a bool that turns on or off the plugin system at large, `plugins.dir` directs the system to a directory to load plugins from, and `plugins.enabled` provides `opt-in` semantics to plugin names to enable (including preloaded plugins). +======= +# gRPC streaming +[streaming] -Configuration of a given plugin is ultimately specific to the plugin, but we will introduce some standards here: +# Turn on/off gRPC streaming +enable = true +>>>>>>> Stashed changes +# List of kv store keys to stream out via gRPC +# Set to ["*"] to expose all keys. +keys = ["*"] + +<<<<<<< Updated upstream Plugin TOML configuration should be split into separate sub-tables for each kind of plugin (e.g. `plugins.streaming`). Within these sub-tables, the parameters for a specific plugin of that kind are included in another sub-table (e.g. `plugins.streaming.file`). @@ -523,8 +812,21 @@ e.g. flush_timeout_ms = 5000 # Flush and wait for outstanding messages and requests to complete delivery when calling `StreamingService.Close(). (milliseconds) halt_app_on_delivery_error = true # Whether or not to halt the application when plugin fails to deliver message(s). ... +======= +# The plugin name used for streaming via gRPC +plugin = "my_streaming_plugin" + +# Stop node on deliver error. +# When false, the node will operate in a fire-and-forget mode +# When true, the node will panic with an error. +stop-node-on-err = true +>>>>>>> Stashed changes ``` +There will be four parameters for configuring the streaming plugin system: `streaming.enable`, `streaming.keys`, `streaming.plugin` and `streaming.stop-node-on-err`. +`streaming.enable` is a bool that turns on or off the streaming service, `streaming.keys` is a set of store keys for stores it listens to, +`streaming.plugin` is the name of the plugin we want to use for streaming and `streaming.stop-node-on-err` is a bool that stops the node when true and operates in a fire-and-forget mode when false. + #### Encoding and decoding streams ADR-038 introduces the interfaces and types for streaming state changes out from KVStores, associating this From e62ef74e66ceae291f8fb943f28bae4c335e7454 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj <87285445+egaxhaj@users.noreply.github.com> Date: Thu, 6 Oct 2022 20:26:39 -0500 Subject: [PATCH 02/20] fix typo Co-authored-by: yihuang --- docs/architecture/adr-038-state-listening.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index 2a9e62808e2..e27caf13a7b 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -207,7 +207,7 @@ func (rs *Store) CacheMultiStore() types.CacheMultiStore { #### Streaming service We will introduce a new `StreamingService` struct for exposing `WriteListener` data streams to external consumers. -In addition to streaming state changes as `StoreKVPair`s, the strut satisfies an `ABCIListener` interface that plugs +In addition to streaming state changes as `StoreKVPair`s, the struct satisfies an `ABCIListener` interface that plugs into the BaseApp and relays ABCI requests and responses so that the service can group the state changes with the ABCI requests that affected them and the ABCI responses they affected. The `ABCIListener` interface also exposes a `ListenKVStorePair` method which is used by the `StoreKVPairWriteListener` to stream state change events. From 9835def2613425276237a1391e1dfc1eba3c80a2 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj <87285445+egaxhaj@users.noreply.github.com> Date: Fri, 7 Oct 2022 11:07:34 -0500 Subject: [PATCH 03/20] remove confict Co-authored-by: Marko --- docs/architecture/adr-038-state-listening.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index e27caf13a7b..ee03e09ff37 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -215,7 +215,6 @@ requests that affected them and the ABCI responses they affected. The `ABCIListe ```go // ABCIListener interface used to hook into the ABCI message processing of the BaseApp type ABCIListener interface { -<<<<<<< Updated upstream // ListenBeginBlock updates the streaming service with the latest BeginBlock messages ListenBeginBlock(ctx types.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error // ListenEndBlock updates the steaming service with the latest EndBlock messages From 6f2432922fb358ec8e745cf8ef42d0ed8eba2d8b Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj <87285445+egaxhaj@users.noreply.github.com> Date: Fri, 7 Oct 2022 11:11:06 -0500 Subject: [PATCH 04/20] header formatting Co-authored-by: Aleksandr Bezobchuk --- docs/architecture/adr-038-state-listening.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index ee03e09ff37..841522a3828 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -204,7 +204,7 @@ func (rs *Store) CacheMultiStore() types.CacheMultiStore { ### Exposing the data -#### Streaming service +#### Streaming Service We will introduce a new `StreamingService` struct for exposing `WriteListener` data streams to external consumers. In addition to streaming state changes as `StoreKVPair`s, the struct satisfies an `ABCIListener` interface that plugs From 5a123c6eb0fd53d609e84d9f625d0e35c6ecc848 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Fri, 7 Oct 2022 15:28:24 -0500 Subject: [PATCH 05/20] fix conflicts --- docs/architecture/adr-038-state-listening.md | 210 +------------------ 1 file changed, 2 insertions(+), 208 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index 841522a3828..9e2d1fbde8c 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -215,32 +215,6 @@ requests that affected them and the ABCI responses they affected. The `ABCIListe ```go // ABCIListener interface used to hook into the ABCI message processing of the BaseApp type ABCIListener interface { - // ListenBeginBlock updates the streaming service with the latest BeginBlock messages - ListenBeginBlock(ctx types.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error - // ListenEndBlock updates the steaming service with the latest EndBlock messages - ListenEndBlock(ctx types.Context, req abci.RequestEndBlock, res abci.ResponseEndBlock) error - // ListenDeliverTx updates the steaming service with the latest DeliverTx messages - ListenDeliverTx(ctx types.Context, req abci.RequestDeliverTx, res abci.ResponseDeliverTx) error - // HaltAppOnDeliveryError whether or not to halt the application when delivery of massages fails - // in ListenBeginBlock, ListenEndBlock, ListenDeliverTx. When `false, the app will operate in fire-and-forget mode. - // When `true`, the app will gracefully halt and stop the running node. Uncommitted blocks will - // be replayed to all listeners when the node restarts and all successful listeners that received data - // prior to the halt will receive duplicate data. Whether or not a listener operates in a fire-and-forget mode - // is determined by the listener's configuration property `halt_app_on_delivery_error = true|false`. - HaltAppOnDeliveryError() bool -} - -// StreamingService interface for registering WriteListeners with the BaseApp and updating the service with the ABCI messages using the hooks -type StreamingService interface { - // Stream is the streaming service loop, awaits kv pairs and writes them to a destination stream or file - Stream(wg *sync.WaitGroup) error - // Listeners returns the streaming service's listeners for the BaseApp to register - Listeners() map[types.StoreKey][]store.WriteListener - // ABCIListener interface for hooking into the ABCI messages from inside the BaseApp - ABCIListener - // Closer interface - io.Closer -======= // ListenBeginBlock updates the streaming service with the latest BeginBlock messages ListenBeginBlock(blockHeight int64, req []byte, res []byte) error // ListenEndBlock updates the steaming service with the latest EndBlock messages @@ -364,11 +338,10 @@ func exposeStoreKeys(keysStr []string, keys map[string]*types.KVStoreKey) []type } return exposeStoreKeys ->>>>>>> Stashed changes } ``` -#### BaseApp registration +#### BaseApp Registration We will add a new method to the `BaseApp` to enable the registration of `StreamingService`s: @@ -381,12 +354,8 @@ func (app *BaseApp) SetStreamingService(s StreamingService) { } // register the StreamingService within the BaseApp // BaseApp will pass BeginBlock, DeliverTx, and EndBlock requests and responses to the streaming services to update their ABCI context -<<<<<<< Updated upstream - app.abciListeners = append(app.abciListeners, s) -======= app.abciListener = s.ABCIListener app.stopNodeOnStreamingErr = s.StopNodeOnErr ->>>>>>> Stashed changes } ``` @@ -398,30 +367,6 @@ func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeg ... -<<<<<<< Updated upstream - // call the hooks with the BeginBlock messages - wg := new(sync.WaitGroup) - for _, streamingListener := range app.abciListeners { - streamingListener := streamingListener // https://go.dev/doc/faq#closures_and_goroutines - if streamingListener.HaltAppOnDeliveryError() { - // increment the wait group counter - wg.Add(1) - go func() { - // decrement the counter when the go routine completes - defer wg.Done() - if err := streamingListener.ListenBeginBlock(app.deliverState.ctx, req, res); err != nil { - app.logger.Error("BeginBlock listening hook failed", "height", req.Header.Height, "err", err) - app.halt() - } - }() - } else { - // fire and forget semantics - go func() { - if err := streamingListener.ListenBeginBlock(app.deliverState.ctx, req, res); err != nil { - app.logger.Error("BeginBlock listening hook failed", "height", req.Header.Height, "err", err) - } - }() -======= // call the streaming service hook with the BeginBlock messages if app.abciListener != nil { reqBz, err := req.Marshal() @@ -438,11 +383,8 @@ func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeg if app.stopNodeOnStreamingErr { panic(err) } ->>>>>>> Stashed changes } } - // wait for all the listener calls to finish - wg.Wait() return res ``` @@ -452,34 +394,6 @@ func (app *BaseApp) EndBlock(req abci.RequestEndBlock) (res abci.ResponseEndBloc ... -<<<<<<< Updated upstream - // Call the streaming service hooks with the EndBlock messages - wg := new(sync.WaitGroup) - for _, streamingListener := range app.abciListeners { - streamingListener := streamingListener // https://go.dev/doc/faq#closures_and_goroutines - if streamingListener.HaltAppOnDeliveryError() { - // increment the wait group counter - wg.Add(1) - go func() { - // decrement the counter when the go routine completes - defer wg.Done() - if err := streamingListener.ListenEndBlock(app.deliverState.ctx, req, res); err != nil { - app.logger.Error("EndBlock listening hook failed", "height", req.Height, "err", err) - app.halt() - } - }() - } else { - // fire and forget semantics - go func() { - if err := streamingListener.ListenEndBlock(app.deliverState.ctx, req, res); err != nil { - app.logger.Error("EndBlock listening hook failed", "height", req.Height, "err", err) - } - }() - } - } - // wait for all the listener calls to finish - wg.Wait() -======= // call the streaming service hook with the EndBlock messages if app.abciListener != nil { reqBz, err := req.Marshal() @@ -528,50 +442,11 @@ func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx }() ... ->>>>>>> Stashed changes return res ``` -<<<<<<< Updated upstream -```go -func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx { - - var abciRes abci.ResponseDeliverTx - defer func() { - // call the hooks with the BeginBlock messages - wg := new(sync.WaitGroup) - for _, streamingListener := range app.abciListeners { - streamingListener := streamingListener // https://go.dev/doc/faq#closures_and_goroutines - if streamingListener.HaltAppOnDeliveryError() { - // increment the wait group counter - wg.Add(1) - go func() { - // decrement the counter when the go routine completes - defer wg.Done() - if err := streamingListener.ListenDeliverTx(app.deliverState.ctx, req, abciRes); err != nil { - app.logger.Error("DeliverTx listening hook failed", "err", err) - app.halt() - } - }() - } else { - // fire and forget semantics - go func() { - if err := streamingListener.ListenDeliverTx(app.deliverState.ctx, req, abciRes); err != nil { - app.logger.Error("DeliverTx listening hook failed", "err", err) - } - }() - } - } - // wait for all the listener calls to finish - wg.Wait() - }() - - ... - - return res -======= -#### Go plugin system +#### Go Plugin System We propose a plugin architecture to load and run `StreamingService` and other types of implementations. We will introduce a plugin system over gRPC that is used to load and run Cosmos-SDK plugins. The plugin system uses [hashicorp/go-plugin](https://github.com/hashicorp/go-plugin). @@ -600,7 +475,6 @@ func (p *ListenerGRPCPlugin) GRPCClient( c *grpc.ClientConn, ) (interface{}, error) { return &GRPCClient{client: NewABCIListenerServiceClient(c)}, nil ->>>>>>> Stashed changes } ``` @@ -645,15 +519,6 @@ For the purposes of this ADR we introduce a single state streaming plugin loadin This provides the advantage of using versioned plugins where the plugin interface and gRPC protocol change over time. ```go -<<<<<<< Updated upstream -// StateStreamingPlugin interface for plugins that load a baseapp.StreamingService onto a baseapp.BaseApp -type StateStreamingPlugin interface { - // Register configures and registers the plugin streaming service with the BaseApp - Register(bApp *baseapp.BaseApp, marshaller codec.BinaryCodec, keys map[string]*types.KVStoreKey) error - - // Start starts the background streaming process of the plugin streaming service - Start(wg *sync.WaitGroup) error -======= func NewStreamingPlugin(name string, logLevel string) (interface{}, error) { logger := hclog.New(&hclog.LoggerOptions{ Output: hclog.DefaultOutput, @@ -677,7 +542,6 @@ func NewStreamingPlugin(name string, logLevel string) (interface{}, error) { if err != nil { return nil, err } ->>>>>>> Stashed changes // Request streaming plugin return rpcClient.Dispense(name) @@ -708,34 +572,6 @@ func NewSimApp( evidencetypes.StoreKey, ibctransfertypes.StoreKey, capabilitytypes.StoreKey, ) -<<<<<<< Updated upstream - pluginsOnKey := fmt.Sprintf("%s.%s", plugin.PLUGINS_TOML_KEY, plugin.PLUGINS_ON_TOML_KEY) - if cast.ToBool(appOpts.Get(pluginsOnKey)) { - // this loads the preloaded and any plugins found in `plugins.dir` - pluginLoader, err := loader.NewPluginLoader(appOpts, logger) - if err != nil { - // handle error - } - - // initialize the loaded plugins - if err := pluginLoader.Initialize(); err != nil { - // handle error - } - - // register the plugin(s) with the BaseApp - if err := pluginLoader.Inject(bApp, appCodec, keys); err != nil { - // handle error - } - - // start the plugin services, optionally use wg to synchronize shutdown using io.Closer - wg := new(sync.WaitGroup) - if err := pluginLoader.Start(wg); err != nil { - // handler error - } - } - -======= ->>>>>>> Stashed changes ... // enable streaming @@ -761,57 +597,16 @@ func NewSimApp( The plugin system will be configured within an App's TOML configuration files. ```toml -<<<<<<< Updated upstream -[plugins] - on = false # turn the plugin system, as a whole, on or off - enabled = ["list", "of", "plugin", "names", "to", "enable"] - dir = "the directory to load non-preloaded plugins from; defaults to cosmos-sdk/plugin/plugins" -``` - -There will be three parameters for configuring the plugin system: `plugins.on`, `plugins.enabled` and `plugins.dir`. -`plugins.on` is a bool that turns on or off the plugin system at large, `plugins.dir` directs the system to a directory -to load plugins from, and `plugins.enabled` provides `opt-in` semantics to plugin names to enable (including preloaded plugins). -======= # gRPC streaming [streaming] # Turn on/off gRPC streaming enable = true ->>>>>>> Stashed changes # List of kv store keys to stream out via gRPC # Set to ["*"] to expose all keys. keys = ["*"] -<<<<<<< Updated upstream -Plugin TOML configuration should be split into separate sub-tables for each kind of plugin (e.g. `plugins.streaming`). - -Within these sub-tables, the parameters for a specific plugin of that kind are included in another sub-table (e.g. `plugins.streaming.file`). -It is generally expected, but not required, that a streaming service plugin can be configured with a set of store keys -(e.g. `plugins.streaming.file.keys`) for the stores it listens to and a flag (e.g. `plugins.streaming.file.halt_app_on_delivery_error`) -that signifies whether the service operates in a fire-and-forget capacity, or stop the BaseApp when an error occurs in -any of `ListenBeginBlock`, `ListenEndBlock` and `ListenDeliverTx`. - -e.g. - -```toml -[plugins] - on = false # turn the plugin system, as a whole, on or off - enabled = ["list", "of", "plugin", "names", "to", "enable"] - dir = "the directory to load non-preloaded plugins from; defaults to " - [plugins.streaming] # a mapping of plugin-specific streaming service parameters, mapped to their plugin name - [plugins.streaming.file] # the specific parameters for the file streaming service plugin - keys = ["list", "of", "store", "keys", "we", "want", "to", "expose", "for", "this", "streaming", "service"] - write_dir = "path to the write directory" - prefix = "optional prefix to prepend to the generated file names" - halt_app_on_delivery_error = "false" # false == fire-and-forget; true == stop the application - [plugins.streaming.kafka] - keys = [] - topic_prefix = "block" # Optional prefix for topic names where data will be stored. - flush_timeout_ms = 5000 # Flush and wait for outstanding messages and requests to complete delivery when calling `StreamingService.Close(). (milliseconds) - halt_app_on_delivery_error = true # Whether or not to halt the application when plugin fails to deliver message(s). - ... -======= # The plugin name used for streaming via gRPC plugin = "my_streaming_plugin" @@ -819,7 +614,6 @@ plugin = "my_streaming_plugin" # When false, the node will operate in a fire-and-forget mode # When true, the node will panic with an error. stop-node-on-err = true ->>>>>>> Stashed changes ``` There will be four parameters for configuring the streaming plugin system: `streaming.enable`, `streaming.keys`, `streaming.plugin` and `streaming.stop-node-on-err`. From 9353cc5bc3bef13bceeefd262f69a1e7afadc90e Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Fri, 7 Oct 2022 15:38:45 -0500 Subject: [PATCH 06/20] changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 769c9a67175..dd76a74775f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (cli) [#13147](https://github.com/cosmos/cosmos-sdk/pull/13147) Add the `--append` flag to the `sign-batch` CLI cmd to combine the messages and sign those txs which are created with `--generate-only`. * (x/consensus) [#12905](https://github.com/cosmos/cosmos-sdk/pull/12905) Create a new `x/consensus` module that is now responsible for maintaining Tendermint consensus parameters instead of `x/param`. Legacy types remain in order to facilitate parameter migration from the deprecated `x/params`. App developers should ensure that they execute `baseapp.MigrateParams` during their chain upgrade. These legacy types will be removed in a future release. * (cli) [#13454](https://github.com/cosmos/cosmos-sdk/pull/13454) `sign-batch` CLI can now read multiple transaction files. +* [#13473](https://github.com/cosmos/cosmos-sdk/pull/13473) ADR 038: State Listening - Go Plugin Proposal. ### Improvements From 561b494d6531b8d7f51d9315bcacf6d7d83f5b4d Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Mon, 10 Oct 2022 15:01:57 -0500 Subject: [PATCH 07/20] rename to kvstorewriter to better clerify its function --- docs/architecture/adr-038-state-listening.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index 9e2d1fbde8c..e242b80ff4b 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -236,24 +236,24 @@ type StreamingService struct { } ``` -We will also introduce a new `KVStoreListener` struct as the writer for use in `StoreKVPairWriteListener`. `KVStoreListener` will expose `ABCIListener` for streaming state change events via `ABCIListener.ListenStoreKVPair`, a `BlockHeight` func for determining the block height for which the state change events belong to and a boolean flag for determine whether the node needs to be stopped or not. +We will also introduce a new `KVStoreWriter` struct as the writer for use in `StoreKVPairWriteListener`. `KVStoreWriter` will expose `ABCIListener` for streaming state change events via `ABCIListener.ListenStoreKVPair`, a `BlockHeight` func for determining the block height for which the state change events belong to and a boolean flag for determine whether the node needs to be stopped or not. ```go -// KVStoreListener is used so that we do not need to update the underlying +// KVStoreWriter is used so that we do not need to update the underlying // io.Writer inside the StoreKVPairWriteListener everytime we begin writing -type KVStoreListener struct { +type KVStoreWriter struct { BlockHeight func() int64 listener ABCIListener stopNodeOnErr bool } -// NewKVStoreListener create an instance of an NewKVStoreListener that sends StoreKVPair data to listening service -func NewKVStoreListener( +// NewKVStoreWriter create an instance of a KVStoreWriter that sends StoreKVPair data to listening service +func NewKVStoreWriter( listener ABCIListener, stopNodeOnErr bool, blockHeight func() int64, -) *KVStoreListener { - return &KVStoreListener{ +) *KVStoreWriter { + return &KVStoreWriter{ listener: listener, stopNodeOnErr: stopNodeOnErr, BlockHeight: blockHeight, @@ -261,7 +261,7 @@ func NewKVStoreListener( } // Write satisfies io.Writer -func (iw *KVStoreListener) Write(b []byte) (int, error) { +func (iw *KVStoreWriter) Write(b []byte) (int, error) { blockHeight := iw.BlockHeight() if err := iw.listener.ListenStoreKVPair(blockHeight, b); err != nil { if iw.stopNodeOnErr { @@ -296,7 +296,7 @@ func RegisterStreamingService( stopNodeOnErrKey := fmt.Sprintf("%s.%s", StreamingTomlKey, StreamingStopNodeOnErrTomlKey) stopNodeOnErr := cast.ToBool(appOpts.Get(stopNodeOnErrKey)) blockHeightFn := func() int64 { return bApp.deliverState.ctx.BlockHeight() } - writer := NewKVStoreListener(abciListener, stopNodeOnErr, blockHeightFn) + writer := NewKVStoreWriter(abciListener, stopNodeOnErr, blockHeightFn) listener := types.NewStoreKVPairWriteListener(writer, kodec) listeners := make(map[types.StoreKey][]types.WriteListener, len(exposeStoreKeys)) for _, key := range exposeStoreKeys { From 992db84d99fb96cbddc8f8d6ef82a083154259a4 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Tue, 11 Oct 2022 16:55:46 -0500 Subject: [PATCH 08/20] async listener hooks --- docs/architecture/adr-038-state-listening.md | 163 +++++++++++++------ 1 file changed, 116 insertions(+), 47 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index e242b80ff4b..f7a6c077b18 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -228,7 +228,7 @@ type ABCIListener interface { // StreamingService struct for registering WriteListeners with the BaseApp and updating the service with the ABCI messages using the hooks type StreamingService struct { // Listeners returns the streaming service's listeners for the BaseApp to register - Listeners() map[types.StoreKey][]store.WriteListener + Listeners map[types.StoreKey][]store.WriteListener // ABCIListener interface for hooking into the ABCI messages from inside the BaseApp ABCIListener ABCIListener // StopNodeOnErr stops the node when true @@ -242,7 +242,7 @@ We will also introduce a new `KVStoreWriter` struct as the writer for use in `St // KVStoreWriter is used so that we do not need to update the underlying // io.Writer inside the StoreKVPairWriteListener everytime we begin writing type KVStoreWriter struct { - BlockHeight func() int64 + blockHeight func() int64 listener ABCIListener stopNodeOnErr bool } @@ -256,13 +256,13 @@ func NewKVStoreWriter( return &KVStoreWriter{ listener: listener, stopNodeOnErr: stopNodeOnErr, - BlockHeight: blockHeight, + blockHeight: blockHeight, } } // Write satisfies io.Writer func (iw *KVStoreWriter) Write(b []byte) (int, error) { - blockHeight := iw.BlockHeight() + blockHeight := iw.blockHeight() if err := iw.listener.ListenStoreKVPair(blockHeight, b); err != nil { if iw.stopNodeOnErr { panic(err) @@ -368,25 +368,48 @@ func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeg ... // call the streaming service hook with the BeginBlock messages + wg := new(sync.WaitGroup) if app.abciListener != nil { - reqBz, err := req.Marshal() - if err != nil { - panic(err) - } - resBz, err := res.Marshal() - if err != nil { - panic(err) - } - blockHeight := app.deliverState.ctx.BlockHeight() - if err := app.abciListener.ListenBeginBlock(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("BeginBlock listening hook failed", "height", req.Header.Height, "err", err) - if app.stopNodeOnStreamingErr { - panic(err) - } + if app.stopNodeOnStreamingErr { + wg.Add(1) + go func() { + defer wg.Done() + reqBz, err := req.Marshal() + if err != nil { + panic(err) + } + resBz, err := res.Marshal() + if err != nil { + panic(err) + } + blockHeight := app.deliverState.ctx.BlockHeight() + if err := app.abciListener.ListenBeginBlock(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("BeginBlock listening hook failed", "height", req.Header.Height, "err", err) + panic(err) + } + }() + } else { + go func() { + reqBz, err := req.Marshal() + if err != nil { + panic(err) + } + resBz, err := res.Marshal() + if err != nil { + panic(err) + } + blockHeight := app.deliverState.ctx.BlockHeight() + if err := app.abciListener.ListenBeginBlock(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("BeginBlock listening hook failed", "height", req.Header.Height, "err", err) + } + }() } } + // wait for listener to finish + wg.Wait() return res +} ``` ```go @@ -395,25 +418,48 @@ func (app *BaseApp) EndBlock(req abci.RequestEndBlock) (res abci.ResponseEndBloc ... // call the streaming service hook with the EndBlock messages + wg := new(sync.WaitGroup) if app.abciListener != nil { - reqBz, err := req.Marshal() - if err != nil { - panic(err) - } - resBz, err := res.Marshal() - if err != nil { - panic(err) - } - blockHeight := app.deliverState.ctx.BlockHeight() - if err := app.abciListener.ListenEndBlock(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("EndBlock listening hook failed", "height", req.Height, "err", err) - if app.stopNodeOnStreamingErr { - panic(err) - } + if app.stopNodeOnStreamingErr { + wg.Add(1) + go func() { + defer wg.Done() + reqBz, err := req.Marshal() + if err != nil { + panic(err) + } + resBz, err := res.Marshal() + if err != nil { + panic(err) + } + blockHeight := app.deliverState.ctx.BlockHeight() + if err := app.abciListener.ListenEndBlock(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("EndBlock listening hook failed", "height", req.Height, "err", err) + panic(err) + } + }() + } else { + go func() { + reqBz, err := req.Marshal() + if err != nil { + panic(err) + } + resBz, err := res.Marshal() + if err != nil { + panic(err) + } + blockHeight := app.deliverState.ctx.BlockHeight() + if err := app.abciListener.ListenEndBlock(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("EndBlock listening hook failed", "height", req.Height, "err", err) + } + }() } } + // wait for listener to finish + wg.Wait() return res +} ``` ```go @@ -421,29 +467,52 @@ func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx var abciRes abci.ResponseDeliverTx defer func() { - // call the streaming service hook with the EndBlock messages + // call the streaming service hook with the DeliverTx messages + wg := new(sync.WaitGroup) if app.abciListener != nil { - reqBz, err := req.Marshal() - if err != nil { - panic(err) - } - resBz, err := abciRes.Marshal() - if err != nil { - panic(err) - } - blockHeight := app.deliverState.ctx.BlockHeight() - if err := app.abciListener.ListenDeliverTx(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("DeliverTx listening hook failed", "err", err) - if app.stopNodeOnStreamingErr { - panic(err) - } + if app.stopNodeOnStreamingErr { + wg.Add(1) + go func() { + defer wg.Done() + reqBz, err := req.Marshal() + if err != nil { + panic(err) + } + resBz, err := abciRes.Marshal() + if err != nil { + panic(err) + } + blockHeight := app.deliverState.ctx.BlockHeight() + if err := app.abciListener.ListenDeliverTx(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("DeliverTx listening hook failed", "err", err) + panic(err) + } + }() + } else { + go func() { + reqBz, err := req.Marshal() + if err != nil { + panic(err) + } + resBz, err := abciRes.Marshal() + if err != nil { + panic(err) + } + blockHeight := app.deliverState.ctx.BlockHeight() + if err := app.abciListener.ListenDeliverTx(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("DeliverTx listening hook failed", "err", err) + } + }() } } + // wait for listener to finish + wg.Wait() }() ... return res +} ``` #### Go Plugin System From 9ab38aca49b996bdfdfa619ec87b97a574377fa2 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Wed, 12 Oct 2022 20:34:11 -0500 Subject: [PATCH 09/20] remove goroutine on blocking calls --- docs/architecture/adr-038-state-listening.md | 163 +++++++++---------- 1 file changed, 74 insertions(+), 89 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index f7a6c077b18..4b13720f34c 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -368,45 +368,40 @@ func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeg ... // call the streaming service hook with the BeginBlock messages - wg := new(sync.WaitGroup) if app.abciListener != nil { + blockHeight := app.deliverState.ctx.BlockHeight() if app.stopNodeOnStreamingErr { - wg.Add(1) - go func() { - defer wg.Done() - reqBz, err := req.Marshal() - if err != nil { - panic(err) - } - resBz, err := res.Marshal() - if err != nil { - panic(err) - } - blockHeight := app.deliverState.ctx.BlockHeight() - if err := app.abciListener.ListenBeginBlock(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("BeginBlock listening hook failed", "height", req.Header.Height, "err", err) - panic(err) - } - }() + reqBz, err := req.Marshal() + if err != nil { + panic(err) + } + resBz, err := res.Marshal() + if err != nil { + panic(err) + } + if err := app.abciListener.ListenBeginBlock(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", err) + panic(err) + } } else { + ebReq, ebRes := req, res go func() { - reqBz, err := req.Marshal() - if err != nil { - panic(err) + reqBz, reqErr := ebReq.Marshal() + if reqErr != nil { + app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", reqErr) } - resBz, err := res.Marshal() - if err != nil { - panic(err) + resBz, resErr := ebRes.Marshal() + if resErr != nil { + app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", resErr) } - blockHeight := app.deliverState.ctx.BlockHeight() - if err := app.abciListener.ListenBeginBlock(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("BeginBlock listening hook failed", "height", req.Header.Height, "err", err) + if reqErr == nil && resErr == nil { + if err := app.abciListener.ListenBeginBlock(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", err) + } } }() } } - // wait for listener to finish - wg.Wait() return res } @@ -418,45 +413,40 @@ func (app *BaseApp) EndBlock(req abci.RequestEndBlock) (res abci.ResponseEndBloc ... // call the streaming service hook with the EndBlock messages - wg := new(sync.WaitGroup) if app.abciListener != nil { + blockHeight := app.deliverState.ctx.BlockHeight() if app.stopNodeOnStreamingErr { - wg.Add(1) - go func() { - defer wg.Done() - reqBz, err := req.Marshal() - if err != nil { - panic(err) - } - resBz, err := res.Marshal() - if err != nil { - panic(err) - } - blockHeight := app.deliverState.ctx.BlockHeight() - if err := app.abciListener.ListenEndBlock(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("EndBlock listening hook failed", "height", req.Height, "err", err) - panic(err) - } - }() + reqBz, err := req.Marshal() + if err != nil { + panic(err) + } + resBz, err := res.Marshal() + if err != nil { + panic(err) + } + if err := app.abciListener.ListenEndBlock(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", err) + panic(err) + } } else { + ebReq, ebRes := req, res go func() { - reqBz, err := req.Marshal() - if err != nil { - panic(err) + reqBz, reqErr := ebReq.Marshal() + if reqErr != nil { + app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", reqErr) } - resBz, err := res.Marshal() - if err != nil { - panic(err) + resBz, resErr := ebRes.Marshal() + if resErr != nil { + app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", resErr) } - blockHeight := app.deliverState.ctx.BlockHeight() - if err := app.abciListener.ListenEndBlock(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("EndBlock listening hook failed", "height", req.Height, "err", err) + if reqErr == nil && resErr == nil { + if err := app.abciListener.ListenEndBlock(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", err) + } } }() } } - // wait for listener to finish - wg.Wait() return res } @@ -467,51 +457,46 @@ func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx var abciRes abci.ResponseDeliverTx defer func() { - // call the streaming service hook with the DeliverTx messages - wg := new(sync.WaitGroup) + // call the streaming service hook with the EndBlock messages if app.abciListener != nil { + blockHeight := app.deliverState.ctx.BlockHeight() if app.stopNodeOnStreamingErr { - wg.Add(1) - go func() { - defer wg.Done() - reqBz, err := req.Marshal() - if err != nil { - panic(err) - } - resBz, err := abciRes.Marshal() - if err != nil { - panic(err) - } - blockHeight := app.deliverState.ctx.BlockHeight() - if err := app.abciListener.ListenDeliverTx(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("DeliverTx listening hook failed", "err", err) - panic(err) - } - }() + reqBz, err := req.Marshal() + if err != nil { + panic(err) + } + resBz, err := abciRes.Marshal() + if err != nil { + panic(err) + } + if err := app.abciListener.ListenDeliverTx(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", err) + panic(err) + } } else { + txReq, txRes := req, abciRes go func() { - reqBz, err := req.Marshal() - if err != nil { - panic(err) + reqBz, reqErr := txReq.Marshal() + if reqErr != nil { + app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", reqErr) } - resBz, err := abciRes.Marshal() - if err != nil { - panic(err) + resBz, resErr := txRes.Marshal() + if resErr != nil { + app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", resErr) } - blockHeight := app.deliverState.ctx.BlockHeight() - if err := app.abciListener.ListenDeliverTx(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("DeliverTx listening hook failed", "err", err) + if reqErr == nil && resErr == nil { + if err := app.abciListener.ListenDeliverTx(blockHeight, reqBz, resBz); err != nil { + app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", err) + } } }() } } - // wait for listener to finish - wg.Wait() }() ... - return res + return abciRes } ``` From afa5702f8d20bce42593d9ad6811fa145cedf6a2 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Thu, 13 Oct 2022 11:04:29 -0500 Subject: [PATCH 10/20] terminate with os.Exit --- docs/architecture/adr-038-state-listening.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index 4b13720f34c..9e9ebf7470f 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -373,15 +373,15 @@ func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeg if app.stopNodeOnStreamingErr { reqBz, err := req.Marshal() if err != nil { - panic(err) + os.Exit(1) } resBz, err := res.Marshal() if err != nil { - panic(err) + os.Exit(1) } if err := app.abciListener.ListenBeginBlock(blockHeight, reqBz, resBz); err != nil { app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", err) - panic(err) + os.Exit(1) } } else { ebReq, ebRes := req, res @@ -418,15 +418,15 @@ func (app *BaseApp) EndBlock(req abci.RequestEndBlock) (res abci.ResponseEndBloc if app.stopNodeOnStreamingErr { reqBz, err := req.Marshal() if err != nil { - panic(err) + os.Exit(1) } resBz, err := res.Marshal() if err != nil { - panic(err) + os.Exit(1) } if err := app.abciListener.ListenEndBlock(blockHeight, reqBz, resBz); err != nil { app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", err) - panic(err) + os.Exit(1) } } else { ebReq, ebRes := req, res @@ -463,15 +463,15 @@ func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx if app.stopNodeOnStreamingErr { reqBz, err := req.Marshal() if err != nil { - panic(err) + os.Exit(1) } resBz, err := abciRes.Marshal() if err != nil { - panic(err) + os.Exit(1) } if err := app.abciListener.ListenDeliverTx(blockHeight, reqBz, resBz); err != nil { app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", err) - panic(err) + os.Exit(1) } } else { txReq, txRes := req, abciRes From 7dc7556a57cabbc3717621804cec83804813ad73 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Mon, 31 Oct 2022 16:14:15 -0500 Subject: [PATCH 11/20] updates from review feedback --- docs/architecture/adr-038-state-listening.md | 844 +++++++++---------- 1 file changed, 403 insertions(+), 441 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index 9e9ebf7470f..8a34d580e79 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -3,6 +3,8 @@ ## Changelog * 11/23/2020: Initial draft +* 10/06/2022: Introduce plugin system based on hashicorp/go-plugin +* 10/30/2022: Updated ADR based on review feedback, move listening to `CommitMultiStore` and add a `MemoryListener` [#13516](https://github.com/cosmos/cosmos-sdk/pull/13516) ## Status @@ -20,31 +22,45 @@ In addition to these request/response queries, it would be beneficial to have a ## Decision -We will modify the `MultiStore` interface and its concrete (`rootmulti` and `cachemulti`) implementations and introduce a new `listenkv.Store` to allow listening to state changes in underlying KVStores. +We will modify the `CommitMultiStore` interface and its concrete (`rootmulti`) implementations and introduce a new `listenkv.Store` to allow listening to state changes in underlying KVStores. We will introduce a plugin system for configuring and running streaming services that write these state changes and their surrounding ABCI message context to different destinations. -### Listening interface +### Listening -In a new file, `store/types/listening.go`, we will create a `WriteListener` interface for streaming out state changes from a KVStore. +In a new file, `store/types/listening.go`, we will create a `MemoryListener` struct for streaming out protobuf encoded KV pairs state changes from a KVStore. +The `MemoryListener` will be used internally by the concrete `rootmulti` implementation to collect state changes from KVStores. ```go -// WriteListener interface for streaming data out from a listenkv.Store -type WriteListener interface { - // if value is nil then it was deleted - // storeKey indicates the source KVStore, to facilitate using the same WriteListener across separate KVStores - // delete bool indicates if it was a delete; true: delete, false: set - OnWrite(storeKey StoreKey, key []byte, value []byte, delete bool) error +// MemoryListener listens to the state writes and accumulate the records in memory. +type MemoryListener struct { + stateCache []StoreKVPair +} + +// NewMemoryListener creates a listener that accumulate the state writes in memory. +func NewMemoryListener() *MemoryListener { + return &MemoryListener{} } -``` -### Listener type +// OnWrite implements MemoryListener interface +func (fl *MemoryListener) OnWrite(storeKey StoreKey, key []byte, value []byte, delete bool) { + fl.stateCache = append(fl.stateCache, StoreKVPair{ + StoreKey: storeKey.Name(), + Delete: delete, + Key: key, + Value: value, + }) +} -We will create a concrete implementation of the `WriteListener` interface in `store/types/listening.go`, that writes out protobuf -encoded KV pairs to an underlying `io.Writer`. +// PopStateCache returns the current state caches and set to nil +func (fl *MemoryListener) PopStateCache() []StoreKVPair { + res := fl.stateCache + fl.stateCache = nil + return res +} +``` -This will include defining a simple protobuf type for the KV pairs. In addition to the key and value fields this message -will include the StoreKey for the originating KVStore so that we can write out from separate KVStores to the same stream/file -and determine the source of each KV pair. +We will also define a protobuf type for the KV pairs. In addition to the key and value fields this message +will include the StoreKey for the originating KVStore so that we can collect information from separate KVStores and determine the source of each KV pair. ```protobuf message StoreKVPair { @@ -55,44 +71,10 @@ message StoreKVPair { } ``` -```go -// StoreKVPairWriteListener is used to configure listening to a KVStore by writing out length-prefixed -// protobuf encoded StoreKVPairs to an underlying io.Writer -type StoreKVPairWriteListener struct { - writer io.Writer - marshaller codec.BinaryCodec -} - -// NewStoreKVPairWriteListener wraps creates a StoreKVPairWriteListener with a provdied io.Writer and codec.BinaryCodec -func NewStoreKVPairWriteListener(w io.Writer, m codec.BinaryCodec) *StoreKVPairWriteListener { - return &StoreKVPairWriteListener{ - writer: w, - marshaller: m, - } -} - -// OnWrite satisfies the WriteListener interface by writing length-prefixed protobuf encoded StoreKVPairs -func (wl *StoreKVPairWriteListener) OnWrite(storeKey types.StoreKey, key []byte, value []byte, delete bool) error error { - kvPair := new(types.StoreKVPair) - kvPair.StoreKey = storeKey.Name() - kvPair.Delete = Delete - kvPair.Key = key - kvPair.Value = value - by, err := wl.marshaller.MarshalBinaryLengthPrefixed(kvPair) - if err != nil { - return err - } - if _, err := wl.writer.Write(by); err != nil { - return err - } - return nil -} -``` - ### ListenKVStore -We will create a new `Store` type `listenkv.Store` that the `MultiStore` wraps around a `KVStore` to enable state listening. -We can configure the `Store` with a set of `WriteListener`s which stream the output to specific destinations. +We will create a new `Store` type `listenkv.Store` that the `rootmulti` store will use to wrap a `KVStore` to enable state listening. +We will configure the `Store` with a `MemoryListener` which will collect state changes for output to specific destinations. ```go // Store implements the KVStore interface with listening enabled. @@ -100,14 +82,14 @@ We can configure the `Store` with a set of `WriteListener`s which stream the out // underlying listeners with the proper key and operation permissions type Store struct { parent types.KVStore - listeners []types.WriteListener + listener *types.MemoryListener parentStoreKey types.StoreKey } // NewStore returns a reference to a new traceKVStore given a parent // KVStore implementation and a buffered writer. -func NewStore(parent types.KVStore, psk types.StoreKey, listeners []types.WriteListener) *Store { - return &Store{parent: parent, listeners: listeners, parentStoreKey: psk} +func NewStore(parent types.KVStore, psk types.StoreKey, listener *types.MemoryListener) *Store { + return &Store{parent: parent, listener: listener, parentStoreKey: psk} } // Set implements the KVStore interface. It traces a write operation and @@ -127,52 +109,31 @@ func (s *Store) Delete(key []byte) { // onWrite writes a KVStore operation to all the WriteListeners func (s *Store) onWrite(delete bool, key, value []byte) { - for _, l := range s.listeners { - if err := l.OnWrite(s.parentStoreKey, key, value, delete); err != nil { - // log error - } - } + s.listener.OnWrite(s.parentStoreKey, key, value, delete) } ``` ### MultiStore interface updates -We will update the `MultiStore` interface to allow us to wrap a set of listeners around a specific `KVStore`. -Additionally, we will update the `CacheWrap` and `CacheWrapper` interfaces to enable listening in the caching layer. +We will update the `CommitMultiStore` interface to allow us to wrap a `Memorylistener` to a specific `KVStore`. +Note that the `MemoryListener` will be attached internally by the concrete `rootmulti` implementation. ```go -type MultiStore interface { +type CommitMultiStore interface { ... - // ListeningEnabled returns if listening is enabled for the KVStore belonging the provided StoreKey - ListeningEnabled(key StoreKey) bool + // AddListeners adds a listener for the KVStore belonging to the provided StoreKey + AddListeners(keys []StoreKey) - // AddListeners adds WriteListeners for the KVStore belonging to the provided StoreKey - // It appends the listeners to a current set, if one already exists - AddListeners(key StoreKey, listeners []WriteListener) + // PopStateCache returns the accumulated state change messages from MemoryListener + PopStateCache() []StoreKVPair } ``` -```go -type CacheWrap interface { - ... - - // CacheWrapWithListeners recursively wraps again with listening enabled - CacheWrapWithListeners(storeKey types.StoreKey, listeners []WriteListener) CacheWrap -} - -type CacheWrapper interface { - ... - - // CacheWrapWithListeners recursively wraps again with listening enabled - CacheWrapWithListeners(storeKey types.StoreKey, listeners []WriteListener) CacheWrap -} -``` ### MultiStore implementation updates -We will modify all of the `Store` and `MultiStore` implementations to satisfy these new interfaces, and adjust the `rootmulti` `GetKVStore` method -to wrap the returned `KVStore` with a `listenkv.Store` if listening is turned on for that `Store`. +We will adjust the `rootmulti` `GetKVStore` method to wrap the returned `KVStore` with a `listenkv.Store` if listening is turned on for that `Store`. ```go func (rs *Store) GetKVStore(key types.StoreKey) types.KVStore { @@ -182,327 +143,248 @@ func (rs *Store) GetKVStore(key types.StoreKey) types.KVStore { store = tracekv.NewStore(store, rs.traceWriter, rs.traceContext) } if rs.ListeningEnabled(key) { - store = listenkv.NewStore(key, store, rs.listeners[key]) + store = listenkv.NewStore(store, key, rs.listeners[key]) } return store } ``` -We will also adjust the `cachemulti` constructor methods and the `rootmulti` `CacheMultiStore` method to forward the listeners -to and enable listening in the cache layer. +We will implement `AddListeners` to manage KVStore listeners internally and implement `PopStateCache` +for a means of retrieving the current state. ```go -func (rs *Store) CacheMultiStore() types.CacheMultiStore { - stores := make(map[types.StoreKey]types.CacheWrapper) - for k, v := range rs.stores { - stores[k] = v - } - return cachemulti.NewStore(rs.db, stores, rs.keysByName, rs.traceWriter, rs.traceContext, rs.listeners) +// AddListeners adds state change listener for a specific KVStore +func (rs *Store) AddListeners(keys []types.StoreKey) { + listener := types.NewMemoryListener() + for i := range keys { + rs.listeners[keys[i]] = listener + } } ``` -### Exposing the data - -#### Streaming Service - -We will introduce a new `StreamingService` struct for exposing `WriteListener` data streams to external consumers. -In addition to streaming state changes as `StoreKVPair`s, the struct satisfies an `ABCIListener` interface that plugs -into the BaseApp and relays ABCI requests and responses so that the service can group the state changes with the ABCI -requests that affected them and the ABCI responses they affected. The `ABCIListener` interface also exposes a -`ListenKVStorePair` method which is used by the `StoreKVPairWriteListener` to stream state change events. - ```go -// ABCIListener interface used to hook into the ABCI message processing of the BaseApp -type ABCIListener interface { - // ListenBeginBlock updates the streaming service with the latest BeginBlock messages - ListenBeginBlock(blockHeight int64, req []byte, res []byte) error - // ListenEndBlock updates the steaming service with the latest EndBlock messages - ListenEndBlock(blockHeight int64, req []byte, res []byte) error - // ListenDeliverTx updates the steaming service with the latest DeliverTx messages - ListenDeliverTx(blockHeight int64, req []byte, res []byte) error - // ListenStoreKVPair updates the steaming service with the latest StoreKVPair messages - ListenStoreKVPair(blockHeight int64, data []byte) error -} - -// StreamingService struct for registering WriteListeners with the BaseApp and updating the service with the ABCI messages using the hooks -type StreamingService struct { - // Listeners returns the streaming service's listeners for the BaseApp to register - Listeners map[types.StoreKey][]store.WriteListener - // ABCIListener interface for hooking into the ABCI messages from inside the BaseApp - ABCIListener ABCIListener - // StopNodeOnErr stops the node when true - StopNodeOnErr bool +func (rs *Store) PopStateCache() []types.StoreKVPair { + var cache []types.StoreKVPair + for _, ls := range rs.listeners { + cache = append(cache, ls.PopStateCache()...) + } + return cache } ``` -We will also introduce a new `KVStoreWriter` struct as the writer for use in `StoreKVPairWriteListener`. `KVStoreWriter` will expose `ABCIListener` for streaming state change events via `ABCIListener.ListenStoreKVPair`, a `BlockHeight` func for determining the block height for which the state change events belong to and a boolean flag for determine whether the node needs to be stopped or not. +We will also adjust the `rootmulti` `CacheMultiStore` and `CacheMultiStoreWithVersion` methods to enable listening in +the cache layer. ```go -// KVStoreWriter is used so that we do not need to update the underlying -// io.Writer inside the StoreKVPairWriteListener everytime we begin writing -type KVStoreWriter struct { - blockHeight func() int64 - listener ABCIListener - stopNodeOnErr bool -} - -// NewKVStoreWriter create an instance of a KVStoreWriter that sends StoreKVPair data to listening service -func NewKVStoreWriter( - listener ABCIListener, - stopNodeOnErr bool, - blockHeight func() int64, -) *KVStoreWriter { - return &KVStoreWriter{ - listener: listener, - stopNodeOnErr: stopNodeOnErr, - blockHeight: blockHeight, - } -} - -// Write satisfies io.Writer -func (iw *KVStoreWriter) Write(b []byte) (int, error) { - blockHeight := iw.blockHeight() - if err := iw.listener.ListenStoreKVPair(blockHeight, b); err != nil { - if iw.stopNodeOnErr { - panic(err) - } - return 0, err - } - return len(b), nil +func (rs *Store) CacheMultiStore() types.CacheMultiStore { + stores := make(map[types.StoreKey]types.CacheWrapper) + for k, v := range rs.stores { + store := v.(types.KVStore) + // Wire the listenkv.Store to allow listeners to observe the writes from the cache store, + // set same listeners on cache store will observe duplicated writes. + if rs.ListeningEnabled(k) { + store = listenkv.NewStore(store, k, rs.listeners[k]) + } + stores[k] = store + } + return cachemulti.NewStore(rs.db, stores, rs.keysByName, rs.traceWriter, rs.getTracingContext()) } ``` -We will expose a `RegisterStreamingService` function for the App to register streaming services. ```go -// RegisterStreamingService registers the ABCI streaming service provided by the streaming plugin. -func RegisterStreamingService( - bApp *BaseApp, - appOpts servertypes.AppOptions, - kodec codec.BinaryCodec, - keys map[string]*types.KVStoreKey, - streamingService interface{}, -) error { - // type checking - abciListener, ok := streamingService.(ABCIListener) - if !ok { - return fmt.Errorf("failed to register streaming service: failed type check %v", streamingService) - } - - // expose keys - keysKey := fmt.Sprintf("%s.%s", StreamingTomlKey, StreamingKeysTomlKey) - exposeKeysStr := cast.ToStringSlice(appOpts.Get(keysKey)) - exposeStoreKeys := exposeStoreKeys(exposeKeysStr, keys) - stopNodeOnErrKey := fmt.Sprintf("%s.%s", StreamingTomlKey, StreamingStopNodeOnErrTomlKey) - stopNodeOnErr := cast.ToBool(appOpts.Get(stopNodeOnErrKey)) - blockHeightFn := func() int64 { return bApp.deliverState.ctx.BlockHeight() } - writer := NewKVStoreWriter(abciListener, stopNodeOnErr, blockHeightFn) - listener := types.NewStoreKVPairWriteListener(writer, kodec) - listeners := make(map[types.StoreKey][]types.WriteListener, len(exposeStoreKeys)) - for _, key := range exposeStoreKeys { - listeners[key] = append(listeners[key], listener) - } - - bApp.SetStreamingService(StreamingService{ - Listeners: listeners, - ABCIListener: abciListener, - StopNodeOnErr: stopNodeOnErr, - }) +func (rs *Store) CacheMultiStoreWithVersion(version int64) (types.CacheMultiStore, error) { + cachedStores := make(map[types.StoreKey]types.CacheWrapper) + for key, store := range rs.stores { + var cacheStore types.KVStore + switch store.GetStoreType() { + case types.StoreTypeIAVL: + // If the store is wrapped with an inter-block cache, we must first unwrap + // it to get the underlying IAVL store. + store = rs.GetCommitKVStore(key) + + // Attempt to lazy-load an already saved IAVL store version. If the + // version does not exist or is pruned, an error should be returned. + var err error + cacheStore, err = store.(*iavl.Store).GetImmutable(version) + if err != nil { + return nil, err + } + default: + cacheStore = store + } - return nil -} + // Wire the listenkv.Store to allow listeners to observe the writes from the cache store, + // set same listeners on cache store will observe duplicated writes. + if rs.ListeningEnabled(key) { + cacheStore = listenkv.NewStore(cacheStore, key, rs.listeners[key]) + } -func exposeAll(list []string) bool { - for _, ele := range list { - if ele == "*" { - return true - } - } - return false -} - -func exposeStoreKeys(keysStr []string, keys map[string]*types.KVStoreKey) []types.StoreKey { - var exposeStoreKeys []types.StoreKey - if exposeAll(keysStr) { - exposeStoreKeys = make([]types.StoreKey, 0, len(keys)) - for _, storeKey := range keys { - exposeStoreKeys = append(exposeStoreKeys, storeKey) - } - } else { - exposeStoreKeys = make([]types.StoreKey, 0, len(keysStr)) - for _, keyStr := range keysStr { - if storeKey, ok := keys[keyStr]; ok { - exposeStoreKeys = append(exposeStoreKeys, storeKey) - } - } - } + cachedStores[key] = cacheStore + } - return exposeStoreKeys + return cachemulti.NewStore(rs.db, cachedStores, rs.keysByName, rs.traceWriter, rs.getTracingContext()), nil } ``` -#### BaseApp Registration +### Exposing the data -We will add a new method to the `BaseApp` to enable the registration of `StreamingService`s: +#### Streaming Service + +We will introduce a new `ABCIListener` interface that plugs into the BaseApp and relays ABCI requests and responses +so that the service can group the state changes with the ABCI requests. ```go -// SetStreamingService is used to set a streaming service into the BaseApp hooks and load the listeners into the multistore -func (app *BaseApp) SetStreamingService(s StreamingService) { - // add the listeners for each StoreKey - for key, lis := range s.Listeners { - app.cms.AddListeners(key, lis) - } - // register the StreamingService within the BaseApp - // BaseApp will pass BeginBlock, DeliverTx, and EndBlock requests and responses to the streaming services to update their ABCI context - app.abciListener = s.ABCIListener - app.stopNodeOnStreamingErr = s.StopNodeOnErr +// baseapp/streaming.go + +// ABCIListener is the interface that we're exposing as a streaming service. +type ABCIListener interface { + // ListenBeginBlock updates the streaming service with the latest BeginBlock messages + ListenBeginBlock(ctx types.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error + // ListenEndBlock updates the steaming service with the latest EndBlock messages + ListenEndBlock(ctx types.Context, req abci.RequestEndBlock, res abci.ResponseEndBlock) error + // ListenDeliverTx updates the steaming service with the latest DeliverTx messages + ListenDeliverTx(ctx types.Context, req abci.RequestDeliverTx, res abci.ResponseDeliverTx) error + // ListenCommit updates the steaming service with the latest Commit messages and state changes + ListenCommit(ctx types.Context, res abci.ResponseCommit, changeSet []store.StoreKVPair) error } ``` -We will also modify the `BeginBlock`, `EndBlock`, and `DeliverTx` methods to pass ABCI requests and responses to any streaming service hooks registered -with the `BaseApp`. +#### ABCI Event Hooks + +We will modify the `BeginBlock`, `EndBlock`, `DeliverTx` and `Commit` methods to pass ABCI requests and responses +to any streaming service hooks registered with the `BaseApp`. ```go func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeginBlock) { - ... - - // call the streaming service hook with the BeginBlock messages - if app.abciListener != nil { - blockHeight := app.deliverState.ctx.BlockHeight() - if app.stopNodeOnStreamingErr { - reqBz, err := req.Marshal() - if err != nil { - os.Exit(1) - } - resBz, err := res.Marshal() - if err != nil { - os.Exit(1) - } - if err := app.abciListener.ListenBeginBlock(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", err) - os.Exit(1) - } - } else { - ebReq, ebRes := req, res - go func() { - reqBz, reqErr := ebReq.Marshal() - if reqErr != nil { - app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", reqErr) - } - resBz, resErr := ebRes.Marshal() - if resErr != nil { - app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", resErr) - } - if reqErr == nil && resErr == nil { - if err := app.abciListener.ListenBeginBlock(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", err) - } - } - }() - } - } + ... - return res + // call the streaming service hook with the BeginBlock messages + if app.abciListener != nil { + ctx := app.deliverState.ctx + blockHeight := ctx.BlockHeight() + if app.stopNodeOnStreamingErr { + if err := app.abciListener.ListenBeginBlock(ctx, req, res); err != nil { + app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", err) + os.Exit(1) + } + } else { + rek, rez := req, res + go func() { + if err := app.abciListener.ListenBeginBlock(ctx, rek, rez); err != nil { + app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", err) + } + }() + } + } + + return res } ``` ```go func (app *BaseApp) EndBlock(req abci.RequestEndBlock) (res abci.ResponseEndBlock) { - ... - - // call the streaming service hook with the EndBlock messages - if app.abciListener != nil { - blockHeight := app.deliverState.ctx.BlockHeight() - if app.stopNodeOnStreamingErr { - reqBz, err := req.Marshal() - if err != nil { - os.Exit(1) - } - resBz, err := res.Marshal() - if err != nil { - os.Exit(1) - } - if err := app.abciListener.ListenEndBlock(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", err) - os.Exit(1) - } - } else { - ebReq, ebRes := req, res - go func() { - reqBz, reqErr := ebReq.Marshal() - if reqErr != nil { - app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", reqErr) - } - resBz, resErr := ebRes.Marshal() - if resErr != nil { - app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", resErr) - } - if reqErr == nil && resErr == nil { - if err := app.abciListener.ListenEndBlock(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", err) - } - } - }() - } - } + ... - return res + // call the streaming service hook with the EndBlock messages + if app.abciListener != nil { + ctx := app.deliverState.ctx + blockHeight := ctx.BlockHeight() + if app.stopNodeOnStreamingErr { + if err := app.abciListener.ListenEndBlock(blockHeight, req, res); err != nil { + app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", err) + os.Exit(1) + } + } else { + rek, rez := req, res + go func() { + if reqErr == nil && resErr == nil { + if err := app.abciListener.ListenEndBlock(blockHeight, rek, rez); err != nil { + app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", err) + } + } + }() + } + } + + return res } ``` ```go func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx { - var abciRes abci.ResponseDeliverTx - defer func() { - // call the streaming service hook with the EndBlock messages - if app.abciListener != nil { - blockHeight := app.deliverState.ctx.BlockHeight() - if app.stopNodeOnStreamingErr { - reqBz, err := req.Marshal() - if err != nil { - os.Exit(1) - } - resBz, err := abciRes.Marshal() - if err != nil { - os.Exit(1) - } - if err := app.abciListener.ListenDeliverTx(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", err) - os.Exit(1) - } - } else { - txReq, txRes := req, abciRes - go func() { - reqBz, reqErr := txReq.Marshal() - if reqErr != nil { - app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", reqErr) - } - resBz, resErr := txRes.Marshal() - if resErr != nil { - app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", resErr) - } - if reqErr == nil && resErr == nil { - if err := app.abciListener.ListenDeliverTx(blockHeight, reqBz, resBz); err != nil { - app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", err) - } - } - }() - } - } - }() - - ... - - return abciRes + var abciRes abci.ResponseDeliverTx + defer func() { + // call the streaming service hook with the EndBlock messages + if app.abciListener != nil { + ctx := app.deliverState.ctx + blockHeight := ctx.BlockHeight() + if app.stopNodeOnStreamingErr { + if err := app.abciListener.ListenDeliverTx(blockHeight, req, res); err != nil { + app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", err) + os.Exit(1) + } + } else { + rek, rez := req, abciRes + go func() { + if reqErr == nil && resErr == nil { + if err := app.abciListener.ListenDeliverTx(blockHeight, rek, rez); err != nil { + app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", err) + } + } + }() + } + } + }() + + ... + + return abciRes +} +``` + +```go +func (app *BaseApp) Commit() abci.ResponseCommit { + + ... + + res := abci.ResponseCommit{ + Data: commitID.Hash, + RetainHeight: retainHeight, + } + + // call the streaming service hook with the Commit messages + if app.abciListener != nil { + ctx := app.deliverState.ctx + blockHeight := ctx.BlockHeight() + changeSet := app.cms.PopStateCache() + if app.stopNodeOnStreamingErr { + if err := app.abciListener.ListenCommit(ctx, res, changeSet); err != nil { + app.logger.Error("ListenCommit listening hook failed", "height", blockHeight, "err", err) + os.Exit(1) + } + } else { + rez := res + go func() { + if err := app.abciListener.ListenCommit(ctx, rez, changeSet); err != nil { + app.logger.Error("ListenCommit listening hook failed", "height", blockHeight, "err", err) + } + }() + } + } + + ... + + return res } ``` #### Go Plugin System -We propose a plugin architecture to load and run `StreamingService` and other types of implementations. We will introduce a plugin +We propose a plugin architecture to load and run `Streaming` plugins and other types of implementations. We will introduce a plugin system over gRPC that is used to load and run Cosmos-SDK plugins. The plugin system uses [hashicorp/go-plugin](https://github.com/hashicorp/go-plugin). Each plugin must have a struct that implements the `plugin.Plugin` interface and an `Impl` interface for processing messages over gRPC. Each plugin must also have a message protocol defined for the gRPC service: @@ -510,7 +392,7 @@ Each plugin must also have a message protocol defined for the gRPC service: ```go // ListenerPlugin is the base struc for all kinds of go-plugin implementations // It will be included in interfaces of different Plugins -type ListenerPlugin interface { +type ABCIListenerPlugin interface { // GRPCPlugin must still implement the Plugin interface plugin.Plugin // Concrete implementation, written in Go. This is only used for plugins @@ -534,116 +416,198 @@ func (p *ListenerGRPCPlugin) GRPCClient( The `plugin.Plugin` interface has two methods `Client` and `Server`. For our GRPC service these are `GRPCClient` and `GRPCServer` The `Impl` field holds the concrete implementation of our `baseapp.ABCIListener` interface written in Go. -Note: this is only used for plugins that are written in Go. +Note: this is only used for plugin implementations written in Go. -The `StreamingService` protocol: +We will introduce a plugin loading system that will return `(interface{}, error)`. +This provides the advantage of using versioned plugins where the plugin interface and gRPC protocol change over time. +In addition, it allows for building independent plugin that can expose different parts of the system over gRPC. + +```go +func NewStreamingPlugin(name string, logLevel string) (interface{}, error) { + logger := hclog.New(&hclog.LoggerOptions{ + Output: hclog.DefaultOutput, + Level: toHclogLevel(logLevel), + Name: fmt.Sprintf("plugin.%s", name), + }) + + // We're a host. Start by launching the streaming process. + env := os.Getenv(GetPluginEnvKey(name)) + client := plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: HandshakeMap[name], + Plugins: PluginMap, + Cmd: exec.Command("sh", "-c", env), + Logger: logger, + AllowedProtocols: []plugin.Protocol{ + plugin.ProtocolNetRPC, plugin.ProtocolGRPC}, + }) + + // Connect via RPC + rpcClient, err := client.Client() + if err != nil { + return nil, err + } + + // Request streaming plugin + return rpcClient.Dispense(name) +} + +``` + +The advantage of having such a plugin system is that within each plugin, authors can define the message protocol in a way that fits their use case. +For example, when state change listening is desired, the `ABCIListener` message protocol can be defined as below (*for illustrative purposes only*). +When state change listening is not desired than `ListenCommit` can be omitted from the protocol. ```protobuf syntax = "proto3"; -package cosmos.sdk.grpc.abci.v1; +... -option go_package = "github.com/cosmos/cosmos-sdk/streaming/plugins/abci/grpc_abci_v1"; -option java_multiple_files = true; -option java_outer_classname = "AbciListenerProto"; -option java_package = "network.cosmos.sdk.grpc.abci.v1"; +message Empty {} -// PutRequest is used for storing ABCI request and response -// and Store KV data for streaming to external grpc service. -message PutRequest { - int64 block_height = 1; - bytes req = 2; - bytes res = 3; - bytes store_kv_pair = 4; - int64 store_kv_pair_idx = 5; - int64 tx_idx = 6; +message ListenBeginBlockRequest { + RequestBeginBlock req = 1; + ResponseBeginBlock res = 2; +} +message ListenBeginBlockRequest {...} +message ListenDeliverTxRequest {...} +message ListenCommitRequest { + ResponseCommit res = 1; + repeated StoreKVPair changeSet = 2; } -message Empty {} +// plugin that listens to state changes +service ABCIListenerService { + rpc ListenBeginBlock(ListenBeginBlockRequest) returns (Empty); + rpc ListenEndBlock(ListenBeginBlockRequest) returns (Empty); + rpc ListenDeliverTx(ListenDeliverTxRequest) returns (Empty); + rpc ListenCommit(ListenCommitRequest) returns (Empty); +} +``` +```protobuf +... +// plugin that doesn't listen to state changes service ABCIListenerService { - rpc ListenBeginBlock(PutRequest) returns (Empty); - rpc ListenEndBlock(PutRequest) returns (Empty); - rpc ListenDeliverTx(PutRequest) returns (Empty); - rpc ListenStoreKVPair(PutRequest) returns (Empty); + rpc ListenBeginBlock(ListenBeginBlockRequest) returns (Empty); + rpc ListenEndBlock(ListenBeginBlockRequest) returns (Empty); + rpc ListenDeliverTx(ListenDeliverTxRequest) returns (Empty); } ``` -For the purposes of this ADR we introduce a single state streaming plugin loading system. This loading system returns `(interface{}, error)`. -This provides the advantage of using versioned plugins where the plugin interface and gRPC protocol change over time. +We will expose a `RegisterStreamingPlugin` function for the App to register streaming plugins with the App's BaseApp. ```go -func NewStreamingPlugin(name string, logLevel string) (interface{}, error) { - logger := hclog.New(&hclog.LoggerOptions{ - Output: hclog.DefaultOutput, - Level: toHclogLevel(logLevel), - Name: fmt.Sprintf("plugin.%s", name), - }) - - // We're a host. Start by launching the streaming process. - env := os.Getenv(GetPluginEnvKey(name)) - client := plugin.NewClient(&plugin.ClientConfig{ - HandshakeConfig: HandshakeMap[name], - Plugins: PluginMap, - Cmd: exec.Command("sh", "-c", env), - Logger: logger, - AllowedProtocols: []plugin.Protocol{ - plugin.ProtocolNetRPC, plugin.ProtocolGRPC}, - }) +// baseapp/streaming.go + +// RegisterStreamingPlugin registers the ABCI streaming service provided by the streaming plugin. +func RegisterStreamingPlugin( + bApp *BaseApp, + appOpts servertypes.AppOptions, + keys map[string]*types.KVStoreKey, + streamingService interface{}, +) error { + switch t := streamingPlugin.(type) { + case ABCIListener: + registerABCIListenerPlugin(bApp, appOpts, keys, t) + default: + return fmt.Errorf("unexpected plugin type %T", t) + } + return nil +} +``` - // Connect via RPC - rpcClient, err := client.Client() - if err != nil { - return nil, err - } +```go +func registerABCIListenerPlugin( + bApp *BaseApp, + appOpts servertypes.AppOptions, + keys map[string]*store.KVStoreKey, + abciListener ABCIListener, +) { + stopNodeOnErrKey := fmt.Sprintf("%s.%s", StreamingTomlKey, StreamingStopNodeOnErrTomlKey) + stopNodeOnErr := cast.ToBool(appOpts.Get(stopNodeOnErrKey)) + keysKey := fmt.Sprintf("%s.%s", StreamingTomlKey, StreamingKeysTomlKey) + exposeKeysStr := cast.ToStringSlice(appOpts.Get(keysKey)) + bApp.cms.AddListeners(exposeStoreKeysSorted(exposeKeysStr, keys)) + bApp.abciListener = abciListener + bApp.stopNodeOnStreamingErr = stopNodeOnErr +} +``` - // Request streaming plugin - return rpcClient.Dispense(name) +```go +func exposeAll(list []string) bool { + for _, ele := range list { + if ele == "*" { + return true + } + } + return false } +func exposeStoreKeysSorted(keysStr []string, keys map[string]*types.KVStoreKey) []types.StoreKey { + var exposeStoreKeys []types.StoreKey + if exposeAll(keysStr) { + exposeStoreKeys = make([]types.StoreKey, 0, len(keys)) + for _, storeKey := range keys { + exposeStoreKeys = append(exposeStoreKeys, storeKey) + } + } else { + exposeStoreKeys = make([]types.StoreKey, 0, len(keysStr)) + for _, keyStr := range keysStr { + if storeKey, ok := keys[keyStr]; ok { + exposeStoreKeys = append(exposeStoreKeys, storeKey) + } + } + } + // sort storeKeys for deterministic output + sort.SliceStable(exposeStoreKeys, func(i, j int) bool { + return strings.Compare(exposeStoreKeys[i].Name(), exposeStoreKeys[j].Name()) < 0 + }) + + return exposeStoreKeys +} ``` -The `RegisterStreamingService` function is used during App construction to register a plugin with the App's BaseApp using the `NewStreamingPlugin` method. +The `NewStreamingPlugin` and `RegisterStreamingPlugin` functions are used to register a plugin with the App's BaseApp. e.g. in `NewSimApp`: ```go func NewSimApp( - logger log.Logger, - db dbm.DB, - traceStore io.Writer, - loadLatest bool, - appOpts servertypes.AppOptions, - baseAppOptions ...func(*baseapp.BaseApp), + logger log.Logger, + db dbm.DB, + traceStore io.Writer, + loadLatest bool, + appOpts servertypes.AppOptions, + baseAppOptions ...func(*baseapp.BaseApp), ) *SimApp { - ... - - keys := sdk.NewKVStoreKeys( - authtypes.StoreKey, banktypes.StoreKey, stakingtypes.StoreKey, - minttypes.StoreKey, distrtypes.StoreKey, slashingtypes.StoreKey, - govtypes.StoreKey, paramstypes.StoreKey, ibchost.StoreKey, upgradetypes.StoreKey, - evidencetypes.StoreKey, ibctransfertypes.StoreKey, capabilitytypes.StoreKey, - ) - - ... - - // enable streaming - enableKey := fmt.Sprintf("%s.%s", baseapp.StreamingTomlKey, baseapp.StreamingEnableTomlKey) - if enable := cast.ToBool(appOpts.Get(enableKey)); enable { - pluginKey := fmt.Sprintf("%s.%s", baseapp.StreamingTomlKey, baseapp.StreamingPluginTomlKey) - pluginName := cast.ToString(appOpts.Get(pluginKey)) - logLevel := cast.ToString(appOpts.Get(flags.FlagLogLevel)) - plugin, err := streaming.NewStreamingPlugin(pluginName, logLevel) - if err != nil { - tmos.Exit(err.Error()) - } - if err := baseapp.RegisterStreamingService(bApp, appOpts, appCodec, keys, plugin); err != nil { - tmos.Exit(err.Error()) - } - } + ... + + keys := sdk.NewKVStoreKeys( + authtypes.StoreKey, banktypes.StoreKey, stakingtypes.StoreKey, + minttypes.StoreKey, distrtypes.StoreKey, slashingtypes.StoreKey, + govtypes.StoreKey, paramstypes.StoreKey, ibchost.StoreKey, upgradetypes.StoreKey, + evidencetypes.StoreKey, ibctransfertypes.StoreKey, capabilitytypes.StoreKey, + ) + + ... - return app + // enable streaming + enableKey := fmt.Sprintf("%s.%s", baseapp.StreamingTomlKey, baseapp.StreamingEnableTomlKey) + if enable := cast.ToBool(appOpts.Get(enableKey)); enable { + pluginKey := fmt.Sprintf("%s.%s", baseapp.StreamingTomlKey, baseapp.StreamingPluginTomlKey) + pluginName := cast.ToString(appOpts.Get(pluginKey)) + logLevel := cast.ToString(appOpts.Get(flags.FlagLogLevel)) + plugin, err := streaming.NewStreamingPlugin(pluginName, logLevel) + if err != nil { + tmos.Exit(err.Error()) + } + if err := baseapp.RegisterStreamingPlugin(bApp, appOpts, keys, plugin); err != nil { + tmos.Exit(err.Error()) + } + } + + return app ``` #### Configuration @@ -654,19 +618,17 @@ The plugin system will be configured within an App's TOML configuration files. # gRPC streaming [streaming] -# Turn on/off gRPC streaming +# StreamingEnale defines if the gRPC streaming plugins should be enabled. enable = true -# List of kv store keys to stream out via gRPC +# The plugin version to use for ABCI listening +plugin = "grpc_abci_v1" + +# List of kv store keys to listen to for state changes. # Set to ["*"] to expose all keys. keys = ["*"] -# The plugin name used for streaming via gRPC -plugin = "my_streaming_plugin" - -# Stop node on deliver error. -# When false, the node will operate in a fire-and-forget mode -# When true, the node will panic with an error. +# Whether to stop the node on message deliver error. stop-node-on-err = true ``` @@ -688,7 +650,7 @@ These changes will provide a means of subscribing to KVStore state changes in re ### Backwards Compatibility -* This ADR changes the `MultiStore`, `CacheWrap`, and `CacheWrapper` interfaces, implementations supporting the previous version of these interfaces will not support the new ones +* This ADR changes the `CommitMultiStore` interface, implementations supporting the previous version of this interface will not support the new one ### Positive @@ -696,7 +658,7 @@ These changes will provide a means of subscribing to KVStore state changes in re ### Negative -* Changes `MultiStore`, `CacheWrap`, and `CacheWrapper` interfaces +* Changes `CommitMultiStore` interface and its implementations ### Neutral From bc024bb483d9df01268225df537ff989dbba2bc7 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Tue, 1 Nov 2022 11:56:02 -0500 Subject: [PATCH 12/20] update comment, call listener in Set/Delete directly --- docs/architecture/adr-038-state-listening.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index 8a34d580e79..b53907b45d5 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -41,7 +41,7 @@ func NewMemoryListener() *MemoryListener { return &MemoryListener{} } -// OnWrite implements MemoryListener interface +// OnWrite writes state change events to the internal cache func (fl *MemoryListener) OnWrite(storeKey StoreKey, key []byte, value []byte, delete bool) { fl.stateCache = append(fl.stateCache, StoreKVPair{ StoreKey: storeKey.Name(), @@ -97,19 +97,14 @@ func NewStore(parent types.KVStore, psk types.StoreKey, listener *types.MemoryLi func (s *Store) Set(key []byte, value []byte) { types.AssertValidKey(key) s.parent.Set(key, value) - s.onWrite(false, key, value) + s.listener.OnWrite(s.parentStoreKey, key, value, false) } // Delete implements the KVStore interface. It traces a write operation and // delegates the Delete call to the parent KVStore. func (s *Store) Delete(key []byte) { s.parent.Delete(key) - s.onWrite(true, key, nil) -} - -// onWrite writes a KVStore operation to all the WriteListeners -func (s *Store) onWrite(delete bool, key, value []byte) { - s.listener.OnWrite(s.parentStoreKey, key, value, delete) + s.listener.OnWrite(s.parentStoreKey, key, nil, true) } ``` From cf4acdb817397d9f63c95a85436fd85896c1f303 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Tue, 1 Nov 2022 12:55:25 -0500 Subject: [PATCH 13/20] add gRPC client, server and plugin implementation examples --- docs/architecture/adr-038-state-listening.md | 204 ++++++++++++++----- 1 file changed, 156 insertions(+), 48 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index b53907b45d5..e337a90f6fa 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -5,6 +5,7 @@ * 11/23/2020: Initial draft * 10/06/2022: Introduce plugin system based on hashicorp/go-plugin * 10/30/2022: Updated ADR based on review feedback, move listening to `CommitMultiStore` and add a `MemoryListener` [#13516](https://github.com/cosmos/cosmos-sdk/pull/13516) +* 11/01/2022: Add gRPC client, server and plugin implementation examples ## Status @@ -385,27 +386,38 @@ Each plugin must have a struct that implements the `plugin.Plugin` interface and Each plugin must also have a message protocol defined for the gRPC service: ```go +// streaming/plugins/abci/{plugin_version}/interface.go + +// Handshake is a common handshake that is shared by streaming and host. +// This prevents users from executing bad plugins or executing a plugin +// directory. It is a UX feature, not a security feature. +var Handshake = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "ABCI_LISTENER_PLUGIN", + MagicCookieValue: "ef78114d-7bdf-411c-868f-347c99a78345", +} + // ListenerPlugin is the base struc for all kinds of go-plugin implementations // It will be included in interfaces of different Plugins -type ABCIListenerPlugin interface { - // GRPCPlugin must still implement the Plugin interface - plugin.Plugin - // Concrete implementation, written in Go. This is only used for plugins - // that are written in Go. - Impl baseapp.ABCIListener +type ABCIListenerPlugin struct { + // GRPCPlugin must still implement the Plugin interface + plugin.Plugin + // Concrete implementation, written in Go. This is only used for plugins + // that are written in Go. + Impl baseapp.ABCIListener } func (p *ListenerGRPCPlugin) GRPCServer(_ *plugin.GRPCBroker, s *grpc.Server) error { - RegisterABCIListenerServiceServer(s, &GRPCServer{Impl: p.Impl}) - return nil + RegisterABCIListenerServiceServer(s, &GRPCServer{Impl: p.Impl}) + return nil } func (p *ListenerGRPCPlugin) GRPCClient( - _ context.Context, - _ *plugin.GRPCBroker, - c *grpc.ClientConn, + _ context.Context, + _ *plugin.GRPCBroker, + c *grpc.ClientConn, ) (interface{}, error) { - return &GRPCClient{client: NewABCIListenerServiceClient(c)}, nil + return &GRPCClient{client: NewABCIListenerServiceClient(c)}, nil } ``` @@ -413,42 +425,7 @@ The `plugin.Plugin` interface has two methods `Client` and `Server`. For our GRP The `Impl` field holds the concrete implementation of our `baseapp.ABCIListener` interface written in Go. Note: this is only used for plugin implementations written in Go. -We will introduce a plugin loading system that will return `(interface{}, error)`. -This provides the advantage of using versioned plugins where the plugin interface and gRPC protocol change over time. -In addition, it allows for building independent plugin that can expose different parts of the system over gRPC. - -```go -func NewStreamingPlugin(name string, logLevel string) (interface{}, error) { - logger := hclog.New(&hclog.LoggerOptions{ - Output: hclog.DefaultOutput, - Level: toHclogLevel(logLevel), - Name: fmt.Sprintf("plugin.%s", name), - }) - - // We're a host. Start by launching the streaming process. - env := os.Getenv(GetPluginEnvKey(name)) - client := plugin.NewClient(&plugin.ClientConfig{ - HandshakeConfig: HandshakeMap[name], - Plugins: PluginMap, - Cmd: exec.Command("sh", "-c", env), - Logger: logger, - AllowedProtocols: []plugin.Protocol{ - plugin.ProtocolNetRPC, plugin.ProtocolGRPC}, - }) - - // Connect via RPC - rpcClient, err := client.Client() - if err != nil { - return nil, err - } - - // Request streaming plugin - return rpcClient.Dispense(name) -} - -``` - -The advantage of having such a plugin system is that within each plugin, authors can define the message protocol in a way that fits their use case. +The advantage of having such a plugin system is that within each plugin authors can define the message protocol in a way that fits their use case. For example, when state change listening is desired, the `ABCIListener` message protocol can be defined as below (*for illustrative purposes only*). When state change listening is not desired than `ListenCommit` can be omitted from the protocol. @@ -489,6 +466,137 @@ service ABCIListenerService { } ``` +Implementing the service above: +```go +// streaming/plugins/abci/{plugin_version}/grpc.go + +var ( + _ baseapp.ABCIListener = (*GRPCClient)(nil) +) + +// GRPCClient is an implementation of the ABCIListener and ABCIListenerPlugin interfaces that talks over RPC. +type GRPCClient struct { + client ABCIListenerServiceClient +} + +func (m *GRPCClient) ListenBeginBlock(ctx types.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error { + _, err := m.client.ListenBeginBlock(ctx, &ListenBeginBlockRequest{Req: req, Res: res}) + return err +} + +func (m *GRPCClient) ListenEndBlock(ctx types.Context, req abci.RequestEndBlock, res abci.ResponseEndBlock) error { + _, err := m.client.ListenEndBlock(ctx, &ListenEndBlockRequest{Req: req, Res: res}) + return err +} + +func (m *GRPCClient) ListenDeliverTx(ctx types.Context, req abci.RequestDeliverTx, res abci.ResponseDeliverTx) error { + _, err := m.client.ListenDeliverTx(ctx, &ListenDeliverTxRequest{Req: req, Res: res}) + return err +} + +func (m *GRPCClient) ListenCommit(ctx types.Context, res abci.ResponseCommit, changeSet []store.StoreKVPair) error { + _, err := m.client.ListenCommit(ctx, &ListenCommitRequest{Res: res, ChangeSet: changeSet}) + return err +} + +// GRPCServer is the gRPC server that GRPCClient talks to. +type GRPCServer struct { + // This is the real implementation + Impl baseapp.ABCIListener +} + +func (m *GRPCServer) ListenBeginBlock(ctx context.Context, req *ListenBeginBlockRequest) (*Empty, error) { + return &Empty{}, m.Impl.ListenBeginBlock(ctx, req.Req, req.Res) +} + +func (m *GRPCServer) ListenEndBlock(ctx context.Context, req *ListenEndBlockRequest) (*Empty, error) { + return &Empty{}, m.Impl.ListenEndBlock(ctx, req.Req, req.Res) +} + +func (m *GRPCServer) ListenDeliverTx(ctx context.Context, req *ListenDeliverTxRequest) (*Empty, error) { + return &Empty{}, m.Impl.ListenDeliverTx(ctx, req.Req, req.Res) +} + +func (m *GRPCServer) ListenCommit(ctx context.Context, req *ListenCommitRequest) (*Empty, error) { + return &Empty{}, m.Impl.ListenCommit(ctx, req.Res, req.ChangeSet) +} + +``` + +And the pre-compiled Go plugin `Impl`(*this is only used for plugins that are written in Go*): + +```go +// streaming/plugins/abci/{plugin_version}/impl/plugin.go + +// Plugins are pre-compiled and loaded by the plugin system + +// ABCIListener is the implementation of the baseapp.ABCIListener interface +type ABCIListener struct{} + +func (m *ABCIListenerPlugin) ListenBeginBlock(ctx context.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error { + // send data to external system +} + +func (m *ABCIListenerPlugin) ListenEndBlock(ctx context.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error { + // send data to external system +} + +func (m *ABCIListenerPlugin) ListenDeliverTxBlock(ctx context.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error { + // send data to external system +} + +func (m *ABCIListenerPlugin) ListenCommit(ctx context.Context, res abci.ResponseCommit, changeSet []store.StoreKVPair) error { + // send data to external system +} + +func main() { + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: grpc_abci_v1.Handshake, + Plugins: map[string]plugin.Plugin{ + "grpc_plugin_v1": &grpc_abci_v1.ABCIListenerGRPCPlugin{Impl: &ABCIListenerPlugin{}}, + }, + + // A non-nil value here enables gRPC serving for this streaming... + GRPCServer: plugin.DefaultGRPCServer, + }) +} +``` + +We will introduce a plugin loading system that will return `(interface{}, error)`. +This provides the advantage of using versioned plugins where the plugin interface and gRPC protocol change over time. +In addition, it allows for building independent plugin that can expose different parts of the system over gRPC. + +```go +func NewStreamingPlugin(name string, logLevel string) (interface{}, error) { + logger := hclog.New(&hclog.LoggerOptions{ + Output: hclog.DefaultOutput, + Level: toHclogLevel(logLevel), + Name: fmt.Sprintf("plugin.%s", name), + }) + + // We're a host. Start by launching the streaming process. + env := os.Getenv(GetPluginEnvKey(name)) + client := plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: HandshakeMap[name], + Plugins: PluginMap, + Cmd: exec.Command("sh", "-c", env), + Logger: logger, + AllowedProtocols: []plugin.Protocol{ + plugin.ProtocolNetRPC, plugin.ProtocolGRPC}, + }) + + // Connect via RPC + rpcClient, err := client.Client() + if err != nil { + return nil, err + } + + // Request streaming plugin + return rpcClient.Dispense(name) +} + +``` + We will expose a `RegisterStreamingPlugin` function for the App to register streaming plugins with the App's BaseApp. ```go From 3e894d742ed641a6acff9f7479b7012f30045bda Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Wed, 2 Nov 2022 13:32:53 -0500 Subject: [PATCH 14/20] update language on RegisterStreamingPlugin function --- docs/architecture/adr-038-state-listening.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index e337a90f6fa..9cad757ed48 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -6,6 +6,7 @@ * 10/06/2022: Introduce plugin system based on hashicorp/go-plugin * 10/30/2022: Updated ADR based on review feedback, move listening to `CommitMultiStore` and add a `MemoryListener` [#13516](https://github.com/cosmos/cosmos-sdk/pull/13516) * 11/01/2022: Add gRPC client, server and plugin implementation examples +* 11/02/2022: Update language on `RegisterStreamingPlugin` function ## Status @@ -597,7 +598,10 @@ func NewStreamingPlugin(name string, logLevel string) (interface{}, error) { ``` -We will expose a `RegisterStreamingPlugin` function for the App to register streaming plugins with the App's BaseApp. +We propose a `RegisterStreamingPlugin` function for the App to register `NewStreamingPlugin`s with the App's BaseApp. +Streaming plugins can be of `Any` type; therefore, the function takes in an interface vs a concrete type. +For example, we could have plugins of `ABCIListener`, `WasmListener` or `IBCListener`. Note that `RegisterStreamingPluing` function +is helper function and not a requirement. Plugin registration can easily be moved from the App to the BaseApp directly. ```go // baseapp/streaming.go From dbcdb0d4435cacacbbe8285d52aaa44aef8977b5 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Wed, 2 Nov 2022 15:53:03 -0500 Subject: [PATCH 15/20] fix changelog diff --- CHANGELOG.md | 605 +++++++++++++++++++++++++-------------------------- 1 file changed, 291 insertions(+), 314 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b1278eef11..6732de640da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,12 +53,11 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (x/bank) [#11981](https://github.com/cosmos/cosmos-sdk/pull/11981) Create the `SetSendEnabled` endpoint for managing the bank's SendEnabled settings. * (x/auth) [#13210](https://github.com/cosmos/cosmos-sdk/pull/13210) Add `Query/AccountInfo` endpoint for simplified access to basic account info. * (x/consensus) [#12905](https://github.com/cosmos/cosmos-sdk/pull/12905) Create a new `x/consensus` module that is now responsible for maintaining Tendermint consensus parameters instead of `x/param`. Legacy types remain in order to facilitate parameter migration from the deprecated `x/params`. App developers should ensure that they execute `baseapp.MigrateParams` during their chain upgrade. These legacy types will be removed in a future release. -* (client/tx) [#13670](https://github.com/cosmos/cosmos-sdk/pull/13670) Add validation in `BuildUnsignedTx` to prevent simple inclusion of valid mnemonics +* (x/auth) [#13612](https://github.com/cosmos/cosmos-sdk/pull/13612) Add `Query/ModuleAccountByName` endpoint for accessing the module account info by module name. * [#13473](https://github.com/cosmos/cosmos-sdk/pull/13473) ADR 038: State Listening - Go Plugin Proposal. ### Improvements -* [#13619](https://github.com/cosmos/cosmos-sdk/pull/13619) Add new function called LogDeferred to report errors in defers. Use the function in x/bank files. * (tools) [#13603](https://github.com/cosmos/cosmos-sdk/pull/13603) Rename cosmovisor package name to `cosmossdk.io/tools/cosmovisor`. The new tool directory contains Cosmos SDK tools. * (deps) [#13397](https://github.com/cosmos/cosmos-sdk/pull/13397) Bump Go version minimum requirement to `1.19`. * [#13070](https://github.com/cosmos/cosmos-sdk/pull/13070) Migrate from `gogo/protobuf` to `cosmos/gogoproto`. @@ -76,15 +75,14 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#12634](https://github.com/cosmos/cosmos-sdk/pull/12634) Move `sdk.Dec` to math package. * [#12596](https://github.com/cosmos/cosmos-sdk/pull/12596) Remove all imports of the non-existent gogo/protobuf v1.3.3 to ease downstream use and go workspaces. * [#12187](https://github.com/cosmos/cosmos-sdk/pull/12187) Add batch operation for x/nft module. -* [#12455](https://github.com/cosmos/cosmos-sdk/pull/12455) Show attempts count in error for signing. +* [#12455](https://github.com/cosmos/cosmos-sdk/pull/12455) Show attempts count in error for signing. * [#13101](https://github.com/cosmos/cosmos-sdk/pull/13101) Remove weights from `simapp/params` and `testutil/sims`. They are now in their respective modules. * [#12398](https://github.com/cosmos/cosmos-sdk/issues/12398) Refactor all `x` modules to unit-test via mocks and decouple `simapp`. * [#13144](https://github.com/cosmos/cosmos-sdk/pull/13144) Add validator distribution info grpc gateway get endpoint. * [#13168](https://github.com/cosmos/cosmos-sdk/pull/13168) Migrate tendermintdev/proto-builder to ghcr.io. New image `ghcr.io/cosmos/proto-builder:0.8` -* [#13178](https://github.com/cosmos/cosmos-sdk/pull/13178) Add `cosmos.msg.v1.service` protobuf annotation to allow tooling to distinguish between Msg and Query services via reflection. +* [#13178](https://github.com/cosmos/cosmos-sdk/pull/13178) Add `cosmos.msg.v1.service` protobuf annotation to allow tooling to distinguish between Msg and Query services via reflection. * [#13236](https://github.com/cosmos/cosmos-sdk/pull/13236) Integrate Filter Logging * [#13528](https://github.com/cosmos/cosmos-sdk/pull/13528) Update `ValidateMemoDecorator` to only check memo against `MaxMemoCharacters` param when a memo is present. -* [#13651](https://github.com/cosmos/cosmos-sdk/pull/13651) Update `server/config/config.GetConfig` function. ### State Machine Breaking @@ -120,6 +118,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (simapp) [#13378](https://github.com/cosmos/cosmos-sdk/pull/13378) Move `simapp.App` to `runtime.AppI`. * (tx) [#12659](https://github.com/cosmos/cosmos-sdk/pull/12659) Remove broadcast mode `block`. * (db) [#13370](https://github.com/cosmos/cosmos-sdk/pull/13370) remove storev2alpha1, see also https://github.com/cosmos/cosmos-sdk/pull/13371 +* (context) [#13063](https://github.com/cosmos/cosmos-sdk/pull/13063) Update `Context#CacheContext` to automatically emit all events on the parent context's `EventManager`. * (x/bank) [#12706](https://github.com/cosmos/cosmos-sdk/pull/12706) Removed the `testutil` package from the `x/bank/client` package. * (simapp) [#12747](https://github.com/cosmos/cosmos-sdk/pull/12747) Remove `simapp.MakeTestEncodingConfig`. Please use `moduletestutil.MakeTestEncodingConfig` (`types/module/testutil`) in tests instead. * (x/bank) [#12648](https://github.com/cosmos/cosmos-sdk/pull/12648) `NewSendAuthorization` takes a new argument of an optional list of addresses allowed to receive bank assests via authz MsgSend grant. You can pass `nil` for the same behavior as before, i.e. any recipient is allowed. @@ -147,11 +146,11 @@ Ref: https://keepachangelog.com/en/1.0.0/ A SendEnabled query has been added to both GRPC and CLI. * (appModule) Remove `Route`, `QuerierRoute` and `LegacyQuerierHandler` from AppModule Interface. * (x/modules) Remove all LegacyQueries and related code from modules -* (store) [#11825](https://github.com/cosmos/cosmos-sdk/pull/11825) Make extension snapshotter interface safer to use, renamed the util function `WriteExtensionItem` to `WriteExtensionPayload`. +* (store) [#11825](https://github.com/cosmos/cosmos-sdk/pull/11825) Make extension snapshotter interface safer to use, renamed the util function `WriteExtensionItem` to `WriteExtensionPayload`. * (x/genutil)[#12956](https://github.com/cosmos/cosmos-sdk/pull/12956) `genutil.AppModuleBasic` has a new attribute: genesis transaction validation function. The existing validation logic is implemented in `genutiltypes.DefaultMessageValidator`. Use `genutil.NewAppModuleBasic` to create a new genutil Module Basic. * (codec) [#12964](https://github.com/cosmos/cosmos-sdk/pull/12964) `ProtoCodec.MarshalInterface` now returns an error when serializing unregistered types and a subsequent `ProtoCodec.UnmarshalInterface` would fail. * (x/staking) [#12973](https://github.com/cosmos/cosmos-sdk/pull/12973) Removed `stakingkeeper.RandomValidator`. Use `testutil.RandSliceElem(r, sk.GetAllValidators(ctx))` instead. -* (x/gov) [#13160](https://github.com/cosmos/cosmos-sdk/pull/13160) Remove custom marshaling of proposl and voteoption. +* (x/gov) [#13160](https://github.com/cosmos/cosmos-sdk/pull/13160) Remove custom marshaling of proposl and voteoption. * (types) [#13430](https://github.com/cosmos/cosmos-sdk/pull/13430) Remove unused code `ResponseCheckTx` and `ResponseDeliverTx` * (store) [#13529](https://github.com/cosmos/cosmos-sdk/pull/13529) Add method `LatestVersion` to `MultiStore` interface, add method `SetQueryMultiStore` to baesapp to support alternative `MultiStore` implementation for query service. * (pruning) [#13609]](https://github.com/cosmos/cosmos-sdk/pull/13609) Move pruning pacakge to be under store pacakge @@ -170,12 +169,14 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (types) [#12154](https://github.com/cosmos/cosmos-sdk/pull/12154) Add `baseAccountGetter` to avoid invalid account error when create vesting account. * (x/authz) [#12184](https://github.com/cosmos/cosmos-sdk/pull/12184) Fix MsgExec not verifying the validity of nested messages. * (x/staking) [#12303](https://github.com/cosmos/cosmos-sdk/pull/12303) Use bytes instead of string comparison in delete validator queue +* (x/auth/tx) [#12474](https://github.com/cosmos/cosmos-sdk/pull/12474) Remove condition in GetTxsEvent that disallowed multiple equal signs, which would break event queries with base64 strings (i.e. query by signature). * (store/rootmulti) [#12487](https://github.com/cosmos/cosmos-sdk/pull/12487) Fix non-deterministic map iteration. * (sdk/dec_coins) [#12903](https://github.com/cosmos/cosmos-sdk/pull/12903) Fix nil `DecCoin` creation when converting `Coins` to `DecCoins` * (store) [#12945](https://github.com/cosmos/cosmos-sdk/pull/12945) Fix nil end semantics in store/cachekv/iterator when iterating a dirty cache. * (x/gov) [#13051](https://github.com/cosmos/cosmos-sdk/pull/13051) In SubmitPropsal, when a legacy msg fails it's handler call, wrap the error as ErrInvalidProposalContent (instead of ErrNoProposalHandlerExists). * (x/gov) [#13045](https://github.com/cosmos/cosmos-sdk/pull/13045) Fix gov migrations for v3(0.46). * (snapshot) [#13400](https://github.com/cosmos/cosmos-sdk/pull/13400) Fix snapshot checksum issue in golang 1.19. +* (store) [#13530](https://github.com/cosmos/cosmos-sdk/pull/13530) Fix app-hash mismatch if upgrade migration commit is interrupted. ### Deprecated @@ -184,41 +185,18 @@ Ref: https://keepachangelog.com/en/1.0.0/ Setting can be done using MsgSetSendEnabled as a governance proposal. A SendEnabled query has been added to both GRPC and CLI. -## [v0.46.4](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.46.4) - 2022-11-01 - -### Features - -* (x/auth) [#13612](https://github.com/cosmos/cosmos-sdk/pull/13612) Add `Query/ModuleAccountByName` endpoint for accessing the module account info by module name. - -### Improvements - -* (deps) Bump IAVL version to [v0.19.4](https://github.com/cosmos/iavl/releases/tag/v0.19.4). - -### Bug Fixes - -* (x/auth/tx) [#12474](https://github.com/cosmos/cosmos-sdk/pull/12474) Remove condition in GetTxsEvent that disallowed multiple equal signs, which would break event queries with base64 strings (i.e. query by signature). -* (store) [#13530](https://github.com/cosmos/cosmos-sdk/pull/13530) Fix app-hash mismatch if upgrade migration commit is interrupted. - -### CLI Breaking Changes - -* [#13656](https://github.com/cosmos/cosmos-sdk/pull/13659) Rename `server.FlagIAVLFastNode` to `server.FlagDisableIAVLFastNode` for clarity. - -### API Breaking Changes - -* (context) [#13063](https://github.com/cosmos/cosmos-sdk/pull/13063) Update `Context#CacheContext` to automatically emit all events on the parent context's `EventManager`. - ## [v0.46.3](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.46.3) - 2022-10-20 ATTENTION: -This is a security release for the [Dragonberry security advisory](https://forum.cosmos.network/t/ibc-security-advisory-dragonberry/7702). +This is a security release for the [Dragonberry security advisory](https://forum.cosmos.network/t/ibc-security-advisory-dragonberry/7702). All users should upgrade immediately. Users *must* add a replace directive in their go.mod for the new `ics23` package in the SDK: ```go -replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0 +replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v8.0.0 ``` ### Features @@ -267,8 +245,8 @@ replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8 * (x/group) [#13214](https://github.com/cosmos/cosmos-sdk/pull/13214) Add `withdraw-proposal` command to group module's CLI transaction commands. * (x/auth) [#13048](https://github.com/cosmos/cosmos-sdk/pull/13048) Add handling of AccountNumberStoreKeyPrefix to the simulation decoder. * (simapp) [#13107](https://github.com/cosmos/cosmos-sdk/pull/13107) Call `SetIAVLCacheSize` with the configured value in simapp. -* [#13301](https://github.com/cosmos/cosmos-sdk/pull/13301) Keep the balance query endpoint compatible with legacy blocks -* [#13321](https://github.com/cosmos/cosmos-sdk/pull/13321) Add flag to disable fast node migration and usage. +* [#13301](https://github.com/cosmos/cosmos-sdk/pull/13301) Keep the balance query endpoint compatible with legacy blocks +* [#13321](https://github.com/cosmos/cosmos-sdk/pull/13321) Add flag to disable fast node migration and usage. ### Bug Fixes @@ -345,7 +323,7 @@ replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8 * (rosetta) [#11590](https://github.com/cosmos/cosmos-sdk/pull/11590) Add fee suggestion for rosetta and enable offline mode. Also force set events about Fees to Success to pass reconciliation test. * (types) [#11959](https://github.com/cosmos/cosmos-sdk/pull/11959) Added `sdk.Coins.Find` helper method to find a coin by denom. * (upgrade) [#12603](https://github.com/cosmos/cosmos-sdk/pull/12603) feat: Move AppModule.BeginBlock and AppModule.EndBlock to extension interfaces -* (telemetry) [#12405](https://github.com/cosmos/cosmos-sdk/pull/12405) Add *query* calls metric to telemetry. +* (telemetry) [#12405](https://github.com/cosmos/cosmos-sdk/pull/12405) Add _query_ calls metric to telemetry. * (query) [#12253](https://github.com/cosmos/cosmos-sdk/pull/12253) Add `GenericFilteredPaginate` to the `query` package to improve UX. ### API Breaking Changes @@ -370,8 +348,8 @@ replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8 * `NewMultiInfo`, `NewLedgerInfo` to `NewLegacyMultiInfo`, `newLegacyLedgerInfo` respectively. Move them into `legacy_info.go`. * `NewOfflineInfo` to `newLegacyOfflineInfo` and move it to `migration_test.go`. * Return: - _`keyring.Record, error` in `SaveOfflineKey`, `SaveLedgerKey`, `SaveMultiSig`, `Key` and `KeyByAddress`. - _`keyring.Record` instead of `Info` in `NewMnemonic` and `List`. + _`keyring.Record, error` in `SaveOfflineKey`, `SaveLedgerKey`, `SaveMultiSig`, `Key` and `KeyByAddress`. + _`keyring.Record` instead of `Info` in `NewMnemonic` and `List`. * Remove `algo` argument from : * `SaveOfflineKey` * Take `keyring.Record` instead of `Info` as first argument in: @@ -558,8 +536,7 @@ replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8 ### State Machine Breaking -* (x/gov) [#13576](https://github.com/cosmos/cosmos-sdk/pull/13576) Proposals in voting period are tracked in a separate store. -* (baseapp) [#11985](https://github.com/cosmos/cosmos-sdk/pull/11985) Add a `postHandler` to baseapp. This `postHandler` is like antehandler, but is run *after* the `runMsgs` execution. It is in the same store branch that `runMsgs`, meaning that both `runMsgs` and `postHandler` +* (baseapp) [#11985](https://github.com/cosmos/cosmos-sdk/pull/11985) Add a `postHandler` to baseapp. This `postHandler` is like antehandler, but is run _after_ the `runMsgs` execution. It is in the same store branch that `runMsgs`, meaning that both `runMsgs` and `postHandler` * (x/gov) [#11998](https://github.com/cosmos/cosmos-sdk/pull/11998) Tweak the `x/gov` `ModuleAccountInvariant` invariant to ensure deposits are `<=` total module account balance instead of strictly equal. * (x/upgrade) [#11800](https://github.com/cosmos/cosmos-sdk/pull/11800) Fix `GetLastCompleteUpgrade` to properly return the latest upgrade. * [#10564](https://github.com/cosmos/cosmos-sdk/pull/10564) Fix bug when updating allowance inside AllowedMsgAllowance @@ -607,11 +584,11 @@ replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8 ATTENTION: -This is a security release for the [Dragonberry security advisory](https://forum.cosmos.network/t/ibc-security-advisory-dragonberry/7702). +This is a security release for the [Dragonberry security advisory](https://forum.cosmos.network/t/ibc-security-advisory-dragonberry/7702). All users should upgrade immediately. -Users *must* add a replace directive in their go.mod for the new `ics23` package in the SDK: +Users _must_ add a replace directive in their go.mod for the new `ics23` package in the SDK: ```go replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0 @@ -627,7 +604,7 @@ replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8 * [#13323](https://github.com/cosmos/cosmos-sdk/pull/13323) Ensure `withdraw_rewards` rewards are emitted from all actions that result in rewards being withdrawn. * [#13321](https://github.com/cosmos/cosmos-sdk/pull/13321) Add flag to disable fast node migration and usage. * (store) [#13326](https://github.com/cosmos/cosmos-sdk/pull/13326) Implementation of ADR-038 file StreamingService, backport #8664. -* (store) [#13540](https://github.com/cosmos/cosmos-sdk/pull/13540) Default fastnode migration to false to prevent suprises. Operators must enable it, unless they have it enabled already. +* (store) [#13540](https://github.com/cosmos/cosmos-sdk/pull/13540) Default fastnode migration to false to prevent suprises. Operators must enable it, unless they have it enabled already. ### API Breaking Changes @@ -676,7 +653,7 @@ Reverted #12437 due to API breaking changes. * (x/mint) [#12384](https://github.com/cosmos/cosmos-sdk/pull/12384) Ensure `GoalBonded` must be positive when performing `x/mint` parameter validation. * (simapp) [#12437](https://github.com/cosmos/cosmos-sdk/pull/12437) fix the non-determinstic behavior in simulations caused by `GenTx` and check - empty coins slice before it is used to create `banktype.MsgSend`. +empty coins slice before it is used to create `banktype.MsgSend`. * (x/capability) [12818](https://github.com/cosmos/cosmos-sdk/pull/12818) Use fixed length hex for pointer at FwdCapabilityKey. ## [v0.45.6](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.45.6) - 2022-06-28 @@ -870,11 +847,11 @@ Reverted #12437 due to API breaking changes. ATTENTION: -This is a security release for the [Dragonberry security advisory](https://forum.cosmos.network/t/ibc-security-advisory-dragonberry/7702). +This is a security release for the [Dragonberry security advisory](https://forum.cosmos.network/t/ibc-security-advisory-dragonberry/7702). All users should upgrade immediately. -Users *must* add a replace directive in their go.mod for the new `ics23` package in the SDK: +Users _must_ add a replace directive in their go.mod for the new `ics23` package in the SDK: ```go replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0 @@ -895,7 +872,7 @@ replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8 ### Improvements -* (types) [#10630](https://github.com/cosmos/cosmos-sdk/pull/10630) Add an `Events` field to the `TxResponse` type that captures *all* events emitted by a transaction, unlike `Logs` which only contains events emitted during message execution. +* (types) [#10630](https://github.com/cosmos/cosmos-sdk/pull/10630) Add an `Events` field to the `TxResponse` type that captures _all_ events emitted by a transaction, unlike `Logs` which only contains events emitted during message execution. * (x/upgrade) [#10532](https://github.com/cosmos/cosmos-sdk/pull/10532) Add `keeper.DumpUpgradeInfoWithInfoToDisk` to include `Plan.Info` in the upgrade-info file. * (store) [#10544](https://github.com/cosmos/cosmos-sdk/pull/10544) Use the new IAVL iterator structure which significantly improves iterator performance. @@ -1386,7 +1363,7 @@ sure you are aware of any relevant breaking changes. * (x/auth) [#6108](https://github.com/cosmos/cosmos-sdk/pull/6108) `tx sign` command's `--validate-signatures` flag is migrated into a `tx validate-signatures` standalone command. * (x/auth) [#7788](https://github.com/cosmos/cosmos-sdk/pull/7788) Remove `tx auth` subcommands, all auth subcommands exist as `tx ` * (x/genutil) [#6651](https://github.com/cosmos/cosmos-sdk/pull/6651) The `gentx` command has been improved. No longer are `--from` and `--name` flags required. Instead, a single argument, `name`, is required which refers to the key pair in the Keyring. In addition, an optional - `--moniker` flag can be provided to override the moniker found in `config.toml`. + `--moniker` flag can be provided to override the moniker found in `config.toml`. * (x/upgrade) [#7697](https://github.com/cosmos/cosmos-sdk/pull/7697) Rename flag name "--time" to "--upgrade-time", "--info" to "--upgrade-info", to keep it consistent with help message. * **REST / Queriers** * (api) [#6426](https://github.com/cosmos/cosmos-sdk/pull/6426) The ability to start an out-of-process API REST server has now been removed. Instead, the API server is now started in-process along with the application and Tendermint. Configuration options have been added to `app.toml` to enable/disable the API server along with additional HTTP server options. @@ -1399,28 +1376,28 @@ sure you are aware of any relevant breaking changes. * (baseapp) [#6384](https://github.com/cosmos/cosmos-sdk/pull/6384) The `Result.Data` is now a Protocol Buffer encoded binary blob of type `TxData`. The `TxData` contains `Data` which contains a list of Protocol Buffer encoded message data and the corresponding message type. * (client) [#5783](https://github.com/cosmos/cosmos-sdk/issues/5783) Unify all coins representations on JSON client requests for governance proposals. * (crypto) [#7419](https://github.com/cosmos/cosmos-sdk/pull/7419) The SDK doesn't use Tendermint's `crypto.PubKey` - interface anymore, and uses instead it's own `PubKey` interface, defined in `crypto/types`. Replace all instances of - `crypto.PubKey` by `cryptotypes.Pubkey`. + interface anymore, and uses instead it's own `PubKey` interface, defined in `crypto/types`. Replace all instances of + `crypto.PubKey` by `cryptotypes.Pubkey`. * (store/rootmulti) [#6390](https://github.com/cosmos/cosmos-sdk/pull/6390) Proofs of empty stores are no longer supported. * (store/types) [#5730](https://github.com/cosmos/cosmos-sdk/pull/5730) store.types.Cp() is removed in favour of types.CopyBytes(). * (x/auth) [#6054](https://github.com/cosmos/cosmos-sdk/pull/6054) Remove custom JSON marshaling for base accounts as multsigs cannot be bech32 decoded. * (x/auth/vesting) [#6859](https://github.com/cosmos/cosmos-sdk/pull/6859) Custom JSON marshaling of vesting accounts was removed. Vesting accounts are now marshaled using their default proto or amino JSON representation. * (x/bank) [#5785](https://github.com/cosmos/cosmos-sdk/issues/5785) In x/bank errors, JSON strings coerced to valid UTF-8 bytes at JSON marshalling time - are now replaced by human-readable expressions. This change can potentially break compatibility with all those client side tools - that parse log messages. + are now replaced by human-readable expressions. This change can potentially break compatibility with all those client side tools + that parse log messages. * (x/evidence) [#7538](https://github.com/cosmos/cosmos-sdk/pull/7538) The ABCI's `Result.Data` field for - `MsgSubmitEvidence` responses does not contain the raw evidence's hash, but the protobuf encoded - `MsgSubmitEvidenceResponse` struct. + `MsgSubmitEvidence` responses does not contain the raw evidence's hash, but the protobuf encoded + `MsgSubmitEvidenceResponse` struct. * (x/gov) [#7533](https://github.com/cosmos/cosmos-sdk/pull/7533) The ABCI's `Result.Data` field for - `MsgSubmitProposal` responses does not contain a raw binary encoding of the `proposalID`, but the protobuf encoded - `MsgSubmitSubmitProposalResponse` struct. + `MsgSubmitProposal` responses does not contain a raw binary encoding of the `proposalID`, but the protobuf encoded + `MsgSubmitSubmitProposalResponse` struct. * (x/gov) [#6859](https://github.com/cosmos/cosmos-sdk/pull/6859) `ProposalStatus` and `VoteOption` are now JSON serialized using its protobuf name, so expect names like `PROPOSAL_STATUS_DEPOSIT_PERIOD` as opposed to `DepositPeriod`. * (x/staking) [#7499](https://github.com/cosmos/cosmos-sdk/pull/7499) `BondStatus` is now a protobuf `enum` instead - of an `int32`, and JSON serialized using its protobuf name, so expect names like `BOND_STATUS_UNBONDING` as opposed - to `Unbonding`. + of an `int32`, and JSON serialized using its protobuf name, so expect names like `BOND_STATUS_UNBONDING` as opposed + to `Unbonding`. * (x/staking) [#7556](https://github.com/cosmos/cosmos-sdk/pull/7556) The ABCI's `Result.Data` field for - `MsgBeginRedelegate` and `MsgUndelegate` responses does not contain custom binary marshaled `completionTime`, but the - protobuf encoded `MsgBeginRedelegateResponse` and `MsgUndelegateResponse` structs respectively + `MsgBeginRedelegate` and `MsgUndelegate` responses does not contain custom binary marshaled `completionTime`, but the + protobuf encoded `MsgBeginRedelegateResponse` and `MsgUndelegateResponse` structs respectively ### API Breaking Changes @@ -1430,7 +1407,7 @@ sure you are aware of any relevant breaking changes. * (client) [#6290](https://github.com/cosmos/cosmos-sdk/pull/6290) `CLIContext` is renamed to `Context`. `Context` and all related methods have been moved from package context to client. * (client) [#6525](https://github.com/cosmos/cosmos-sdk/pull/6525) Removed support for `indent` in JSON responses. Clients should consider piping to an external tool such as `jq`. * (client) [#8107](https://github.com/cosmos/cosmos-sdk/pull/8107) Renamed `PrintOutput` and `PrintOutputLegacy` - methods of the `context.Client` object to `PrintProto` and `PrintObjectLegacy`. + methods of the `context.Client` object to `PrintProto` and `PrintObjectLegacy`. * (client/flags) [#6632](https://github.com/cosmos/cosmos-sdk/pull/6632) Remove NewCompletionCmd(), the function is now available in tendermint. * (client/input) [#5904](https://github.com/cosmos/cosmos-sdk/pull/5904) Removal of unnecessary `GetCheckPassword`, `PrintPrefixed` functions. * (client/keys) [#5889](https://github.com/cosmos/cosmos-sdk/pull/5889) Rename `NewKeyBaseFromDir()` -> `NewLegacyKeyBaseFromDir()`. @@ -1440,32 +1417,32 @@ sure you are aware of any relevant breaking changes. * (codec) [#6330](https://github.com/cosmos/cosmos-sdk/pull/6330) `codec.RegisterCrypto` has been moved to the `crypto/codec` package and the global `codec.Cdc` Amino instance has been deprecated and moved to the `codec/legacy_global` package. * (codec) [#8080](https://github.com/cosmos/cosmos-sdk/pull/8080) Updated the `codec.Marshaler` interface * Moved `MarshalAny` and `UnmarshalAny` helper functions to `codec.Marshaler` and renamed to `MarshalInterface` and - `UnmarshalInterface` respectively. These functions must take interface as a parameter (not a concrete type nor `Any` - object). Underneath they use `Any` wrapping for correct protobuf serialization. + `UnmarshalInterface` respectively. These functions must take interface as a parameter (not a concrete type nor `Any` + object). Underneath they use `Any` wrapping for correct protobuf serialization. * (crypto) [#6780](https://github.com/cosmos/cosmos-sdk/issues/6780) Move ledger code to its own package. * (crypto/types/multisig) [#6373](https://github.com/cosmos/cosmos-sdk/pull/6373) `multisig.Multisignature` has been renamed to `AminoMultisignature` * (codec) `*codec.LegacyAmino` is now a wrapper around Amino which provides backwards compatibility with protobuf `Any`. ALL legacy code should use `*codec.LegacyAmino` instead of `*amino.Codec` directly * (crypto) [#5880](https://github.com/cosmos/cosmos-sdk/pull/5880) Merge `crypto/keys/mintkey` into `crypto`. * (crypto/hd) [#5904](https://github.com/cosmos/cosmos-sdk/pull/5904) `crypto/keys/hd` moved to `crypto/hd`. * (crypto/keyring): - _ [#5866](https://github.com/cosmos/cosmos-sdk/pull/5866) Rename `crypto/keys/` to `crypto/keyring/`. - _ [#5904](https://github.com/cosmos/cosmos-sdk/pull/5904) `Keybase` -> `Keyring` interfaces migration. `LegacyKeybase` interface is added in order - to guarantee limited backward compatibility with the old Keybase interface for the sole purpose of migrating keys across the new keyring backends. `NewLegacy` - constructor is provided [#5889](https://github.com/cosmos/cosmos-sdk/pull/5889) to allow for smooth migration of keys from the legacy LevelDB based implementation - to new keyring backends. Plus, the package and the new keyring no longer depends on the sdk.Config singleton. Please consult the [package documentation](https://github.com/cosmos/cosmos-sdk/tree/master/crypto/keyring/doc.go) for more - information on how to implement the new `Keyring` interface. \* [#5858](https://github.com/cosmos/cosmos-sdk/pull/5858) Make Keyring store keys by name and address's hexbytes representation. + _ [#5866](https://github.com/cosmos/cosmos-sdk/pull/5866) Rename `crypto/keys/` to `crypto/keyring/`. + _ [#5904](https://github.com/cosmos/cosmos-sdk/pull/5904) `Keybase` -> `Keyring` interfaces migration. `LegacyKeybase` interface is added in order + to guarantee limited backward compatibility with the old Keybase interface for the sole purpose of migrating keys across the new keyring backends. `NewLegacy` + constructor is provided [#5889](https://github.com/cosmos/cosmos-sdk/pull/5889) to allow for smooth migration of keys from the legacy LevelDB based implementation + to new keyring backends. Plus, the package and the new keyring no longer depends on the sdk.Config singleton. Please consult the [package documentation](https://github.com/cosmos/cosmos-sdk/tree/master/crypto/keyring/doc.go) for more + information on how to implement the new `Keyring` interface. \* [#5858](https://github.com/cosmos/cosmos-sdk/pull/5858) Make Keyring store keys by name and address's hexbytes representation. * (export) [#5952](https://github.com/cosmos/cosmos-sdk/pull/5952) `AppExporter` now returns ABCI consensus parameters to be included in marshaled exported state. These parameters must be returned from the application via the `BaseApp`. * (simapp) Deprecating and renaming `MakeEncodingConfig` to `MakeTestEncodingConfig` (both in `simapp` and `simapp/params` packages). * (store) [#5803](https://github.com/cosmos/cosmos-sdk/pull/5803) The `store.CommitMultiStore` interface now includes the new `snapshots.Snapshotter` interface as well. * (types) [#5579](https://github.com/cosmos/cosmos-sdk/pull/5579) The `keepRecent` field has been removed from the `PruningOptions` type. - The `PruningOptions` type now only includes fields `KeepEvery` and `SnapshotEvery`, where `KeepEvery` - determines which committed heights are flushed to disk and `SnapshotEvery` determines which of these - heights are kept after pruning. The `IsValid` method should be called whenever using these options. Methods - `SnapshotVersion` and `FlushVersion` accept a version arugment and determine if the version should be - flushed to disk or kept as a snapshot. Note, `KeepRecent` is automatically inferred from the options - and provided directly the IAVL store. + The `PruningOptions` type now only includes fields `KeepEvery` and `SnapshotEvery`, where `KeepEvery` + determines which committed heights are flushed to disk and `SnapshotEvery` determines which of these + heights are kept after pruning. The `IsValid` method should be called whenever using these options. Methods + `SnapshotVersion` and `FlushVersion` accept a version arugment and determine if the version should be + flushed to disk or kept as a snapshot. Note, `KeepRecent` is automatically inferred from the options + and provided directly the IAVL store. * (types) [#5533](https://github.com/cosmos/cosmos-sdk/pull/5533) Refactored `AppModuleBasic` and `AppModuleGenesis` - to now accept a `codec.JSONMarshaler` for modular serialization of genesis state. + to now accept a `codec.JSONMarshaler` for modular serialization of genesis state. * (types/rest) [#5779](https://github.com/cosmos/cosmos-sdk/pull/5779) Drop unused Parse{Int64OrReturnBadRequest,QueryParamBool}() functions. * **Modules** * (modules) [#7243](https://github.com/cosmos/cosmos-sdk/pull/7243) Rename `RegisterCodec` to `RegisterLegacyAminoCodec` and `codec.New()` is now renamed to `codec.NewLegacyAmino()` @@ -1495,10 +1472,10 @@ sure you are aware of any relevant breaking changes. * `SignatureVerificationGasConsumer` now has the signature: `func(meter sdk.GasMeter, sig signing.SignatureV2, params types.Params) error`. * The `SigVerifiableTx` interface now has a `GetSignaturesV2() ([]signing.SignatureV2, error)` method and no longer has the `GetSignBytes` method. * (x/auth/tx) [#8106](https://github.com/cosmos/cosmos-sdk/pull/8106) change related to missing append functionality in - client transaction signing + client transaction signing * added `overwriteSig` argument to `x/auth/client.SignTx` and `client/tx.Sign` functions. * removed `x/auth/tx.go:wrapper.GetSignatures`. The `wrapper` provides `TxBuilder` functionality, and it's a private - structure. That function was not used at all and it's not exposed through the `TxBuilder` interface. + structure. That function was not used at all and it's not exposed through the `TxBuilder` interface. * (x/bank) [#7327](https://github.com/cosmos/cosmos-sdk/pull/7327) AddCoins and SubtractCoins no longer return a resultingValue and will only return an error. * (x/capability) [#7918](https://github.com/cosmos/cosmos-sdk/pull/7918) Add x/capability safety checks: * All outward facing APIs will now check that capability is not nil and name is not empty before performing any state-machine changes @@ -1506,15 +1483,15 @@ sure you are aware of any relevant breaking changes. * (x/evidence) [#7251](https://github.com/cosmos/cosmos-sdk/pull/7251) New evidence types and light client evidence handling. The module function names changed. * (x/evidence) [#5952](https://github.com/cosmos/cosmos-sdk/pull/5952) Remove APIs for getting and setting `x/evidence` parameters. `BaseApp` now uses a `ParamStore` to manage Tendermint consensus parameters which is managed via the `x/params` `Substore` type. * (x/gov) [#6147](https://github.com/cosmos/cosmos-sdk/pull/6147) The `Content` field on `Proposal` and `MsgSubmitProposal` - is now `Any` in concordance with [ADR 019](docs/architecture/adr-019-protobuf-state-encoding.md) and `GetContent` should now - be used to retrieve the actual proposal `Content`. Also the `NewMsgSubmitProposal` constructor now may return an `error` + is now `Any` in concordance with [ADR 019](docs/architecture/adr-019-protobuf-state-encoding.md) and `GetContent` should now + be used to retrieve the actual proposal `Content`. Also the `NewMsgSubmitProposal` constructor now may return an `error` * (x/ibc) [#6374](https://github.com/cosmos/cosmos-sdk/pull/6374) `VerifyMembership` and `VerifyNonMembership` now take a `specs []string` argument to specify the proof format used for verification. Most SDK chains can simply use `commitmenttypes.GetSDKSpecs()` for this argument. * (x/params) [#5619](https://github.com/cosmos/cosmos-sdk/pull/5619) The `x/params` keeper now accepts a `codec.Marshaller` instead of - a reference to an amino codec. Amino is still used for JSON serialization. + a reference to an amino codec. Amino is still used for JSON serialization. * (x/staking) [#6451](https://github.com/cosmos/cosmos-sdk/pull/6451) `DefaultParamspace` and `ParamKeyTable` in staking module are moved from keeper to types to enforce consistency. * (x/staking) [#7419](https://github.com/cosmos/cosmos-sdk/pull/7419) The `TmConsPubKey` method on ValidatorI has been - removed and replaced instead by `ConsPubKey` (which returns a SDK `cryptotypes.PubKey`) and `TmConsPublicKey` (which - returns a Tendermint proto PublicKey). + removed and replaced instead by `ConsPubKey` (which returns a SDK `cryptotypes.PubKey`) and `TmConsPublicKey` (which + returns a Tendermint proto PublicKey). * (x/staking/types) [#7447](https://github.com/cosmos/cosmos-sdk/issues/7447) Remove bech32 PubKey support: * `ValidatorI` interface update. `GetConsPubKey` renamed to `TmConsPubKey` (consensus public key must be a tendermint key). `TmConsPubKey`, `GetConsAddr` methods return error. * `Validator` update. Methods changed in `ValidatorI` (as described above) and `ToTmValidator` return error. @@ -1523,7 +1500,7 @@ sure you are aware of any relevant breaking changes. * (x/supply) [#6010](https://github.com/cosmos/cosmos-sdk/pull/6010) All `x/supply` types and APIs have been moved to `x/bank`. * [#6409](https://github.com/cosmos/cosmos-sdk/pull/6409) Rename all IsEmpty methods to Empty across the codebase and enforce consistency. * [#6231](https://github.com/cosmos/cosmos-sdk/pull/6231) Simplify `AppModule` interface, `Route` and `NewHandler` methods become only `Route` - and returns a new `Route` type. + and returns a new `Route` type. * (x/slashing) [#6212](https://github.com/cosmos/cosmos-sdk/pull/6212) Remove `Get*` prefixes from key construction functions * (server) [#6079](https://github.com/cosmos/cosmos-sdk/pull/6079) Remove `UpgradeOldPrivValFile` (deprecated in Tendermint Core v0.28). * [#5719](https://github.com/cosmos/cosmos-sdk/pull/5719) Bump Go requirement to 1.14+ @@ -1536,75 +1513,75 @@ sure you are aware of any relevant breaking changes. * **Modules** * (modules) [#5572](https://github.com/cosmos/cosmos-sdk/pull/5572) Separate balance from accounts per ADR 004. - _ Account balances are now persisted and retrieved via the `x/bank` module. - _ Vesting account interface has been modified to account for changes. - _ Callers to `NewBaseVestingAccount` are responsible for verifying account balance in relation to - the original vesting amount. - _ The `SendKeeper` and `ViewKeeper` interfaces in `x/bank` have been modified to account for changes. + _ Account balances are now persisted and retrieved via the `x/bank` module. + _ Vesting account interface has been modified to account for changes. + _ Callers to `NewBaseVestingAccount` are responsible for verifying account balance in relation to + the original vesting amount. + _ The `SendKeeper` and `ViewKeeper` interfaces in `x/bank` have been modified to account for changes. * (x/auth) [#5533](https://github.com/cosmos/cosmos-sdk/pull/5533) Migrate the `x/auth` module to use Protocol Buffers for state - serialization instead of Amino. - _ The `BaseAccount.PubKey` field is now represented as a Bech32 string instead of a `crypto.Pubkey`. - _ `NewBaseAccountWithAddress` now returns a reference to a `BaseAccount`. - _ The `x/auth` module now accepts a `Codec` interface which extends the `codec.Marshaler` interface by - requiring a concrete codec to know how to serialize accounts. - _ The `AccountRetriever` type now accepts a `Codec` in its constructor in order to know how to - serialize accounts. + serialization instead of Amino. + _ The `BaseAccount.PubKey` field is now represented as a Bech32 string instead of a `crypto.Pubkey`. + _ `NewBaseAccountWithAddress` now returns a reference to a `BaseAccount`. + _ The `x/auth` module now accepts a `Codec` interface which extends the `codec.Marshaler` interface by + requiring a concrete codec to know how to serialize accounts. + _ The `AccountRetriever` type now accepts a `Codec` in its constructor in order to know how to + serialize accounts. * (x/bank) [#6518](https://github.com/cosmos/cosmos-sdk/pull/6518) Support for global and per-denomination send enabled flags. * Existing send_enabled global flag has been moved into a Params structure as `default_send_enabled`. * An array of: `{denom: string, enabled: bool}` is added to bank Params to support per-denomination override of global default value. * (x/distribution) [#5610](https://github.com/cosmos/cosmos-sdk/pull/5610) Migrate the `x/distribution` module to use Protocol Buffers for state - serialization instead of Amino. The exact codec used is `codec.HybridCodec` which utilizes Protobuf for binary encoding and Amino - for JSON encoding. - _ `ValidatorHistoricalRewards.ReferenceCount` is now of types `uint32` instead of `uint16`. - _ `ValidatorSlashEvents` is now a struct with `slashevents`. - _ `ValidatorOutstandingRewards` is now a struct with `rewards`. - _ `ValidatorAccumulatedCommission` is now a struct with `commission`. \* The `Keeper` constructor now takes a `codec.Marshaler` instead of a concrete Amino codec. This exact type - provided is specified by `ModuleCdc`. + serialization instead of Amino. The exact codec used is `codec.HybridCodec` which utilizes Protobuf for binary encoding and Amino + for JSON encoding. + _ `ValidatorHistoricalRewards.ReferenceCount` is now of types `uint32` instead of `uint16`. + _ `ValidatorSlashEvents` is now a struct with `slashevents`. + _ `ValidatorOutstandingRewards` is now a struct with `rewards`. + _ `ValidatorAccumulatedCommission` is now a struct with `commission`. \* The `Keeper` constructor now takes a `codec.Marshaler` instead of a concrete Amino codec. This exact type + provided is specified by `ModuleCdc`. * (x/evidence) [#5634](https://github.com/cosmos/cosmos-sdk/pull/5634) Migrate the `x/evidence` module to use Protocol Buffers for state - serialization instead of Amino. - _ The `internal` sub-package has been removed in order to expose the types proto file. - _ The module now accepts a `Codec` interface which extends the `codec.Marshaler` interface by - requiring a concrete codec to know how to serialize `Evidence` types. \* The `MsgSubmitEvidence` message has been removed in favor of `MsgSubmitEvidenceBase`. The application-level - codec must now define the concrete `MsgSubmitEvidence` type which must implement the module's `MsgSubmitEvidence` - interface. + serialization instead of Amino. + _ The `internal` sub-package has been removed in order to expose the types proto file. + _ The module now accepts a `Codec` interface which extends the `codec.Marshaler` interface by + requiring a concrete codec to know how to serialize `Evidence` types. \* The `MsgSubmitEvidence` message has been removed in favor of `MsgSubmitEvidenceBase`. The application-level + codec must now define the concrete `MsgSubmitEvidence` type which must implement the module's `MsgSubmitEvidence` + interface. * (x/evidence) [#5952](https://github.com/cosmos/cosmos-sdk/pull/5952) Remove parameters from `x/evidence` genesis and module state. The `x/evidence` module now solely uses Tendermint consensus parameters to determine of evidence is valid or not. * (x/gov) [#5737](https://github.com/cosmos/cosmos-sdk/pull/5737) Migrate the `x/gov` module to use Protocol - Buffers for state serialization instead of Amino. - _ `MsgSubmitProposal` will be removed in favor of the application-level proto-defined `MsgSubmitProposal` which - implements the `MsgSubmitProposalI` interface. Applications should extend the `NewMsgSubmitProposalBase` type - to define their own concrete `MsgSubmitProposal` types. - _ The module now accepts a `Codec` interface which extends the `codec.Marshaler` interface by - requiring a concrete codec to know how to serialize `Proposal` types. + Buffers for state serialization instead of Amino. + _ `MsgSubmitProposal` will be removed in favor of the application-level proto-defined `MsgSubmitProposal` which + implements the `MsgSubmitProposalI` interface. Applications should extend the `NewMsgSubmitProposalBase` type + to define their own concrete `MsgSubmitProposal` types. + _ The module now accepts a `Codec` interface which extends the `codec.Marshaler` interface by + requiring a concrete codec to know how to serialize `Proposal` types. * (x/mint) [#5634](https://github.com/cosmos/cosmos-sdk/pull/5634) Migrate the `x/mint` module to use Protocol Buffers for state - serialization instead of Amino. \* The `internal` sub-package has been removed in order to expose the types proto file. + serialization instead of Amino. \* The `internal` sub-package has been removed in order to expose the types proto file. * (x/slashing) [#5627](https://github.com/cosmos/cosmos-sdk/pull/5627) Migrate the `x/slashing` module to use Protocol Buffers for state - serialization instead of Amino. The exact codec used is `codec.HybridCodec` which utilizes Protobuf for binary encoding and Amino - for JSON encoding. \* The `Keeper` constructor now takes a `codec.Marshaler` instead of a concrete Amino codec. This exact type - provided is specified by `ModuleCdc`. + serialization instead of Amino. The exact codec used is `codec.HybridCodec` which utilizes Protobuf for binary encoding and Amino + for JSON encoding. \* The `Keeper` constructor now takes a `codec.Marshaler` instead of a concrete Amino codec. This exact type + provided is specified by `ModuleCdc`. * (x/staking) [#6844](https://github.com/cosmos/cosmos-sdk/pull/6844) Validators are now inserted into the unbonding queue based on their unbonding time and height. The relevant keeper APIs are modified to reflect these changes by now also requiring a height. * (x/staking) [#6061](https://github.com/cosmos/cosmos-sdk/pull/6061) Allow a validator to immediately unjail when no signing info is present due to - falling below their minimum self-delegation and never having been bonded. The validator may immediately unjail once they've met their minimum self-delegation. + falling below their minimum self-delegation and never having been bonded. The validator may immediately unjail once they've met their minimum self-delegation. * (x/staking) [#5600](https://github.com/cosmos/cosmos-sdk/pull/5600) Migrate the `x/staking` module to use Protocol Buffers for state - serialization instead of Amino. The exact codec used is `codec.HybridCodec` which utilizes Protobuf for binary encoding and Amino - for JSON encoding. - _ `BondStatus` is now of type `int32` instead of `byte`. - _ Types of `int16` in the `Params` type are now of type `int32`. - _ Every reference of `crypto.Pubkey` in context of a `Validator` is now of type string. `GetPubKeyFromBech32` must be used to get the `crypto.Pubkey`. - _ The `Keeper` constructor now takes a `codec.Marshaler` instead of a concrete Amino codec. This exact type - provided is specified by `ModuleCdc`. + serialization instead of Amino. The exact codec used is `codec.HybridCodec` which utilizes Protobuf for binary encoding and Amino + for JSON encoding. + _ `BondStatus` is now of type `int32` instead of `byte`. + _ Types of `int16` in the `Params` type are now of type `int32`. + _ Every reference of `crypto.Pubkey` in context of a `Validator` is now of type string. `GetPubKeyFromBech32` must be used to get the `crypto.Pubkey`. + _ The `Keeper` constructor now takes a `codec.Marshaler` instead of a concrete Amino codec. This exact type + provided is specified by `ModuleCdc`. * (x/staking) [#7979](https://github.com/cosmos/cosmos-sdk/pull/7979) keeper pubkey storage serialization migration - from bech32 to protobuf. + from bech32 to protobuf. * (x/supply) [#6010](https://github.com/cosmos/cosmos-sdk/pull/6010) Removed the `x/supply` module by merging the existing types and APIs into the `x/bank` module. * (x/supply) [#5533](https://github.com/cosmos/cosmos-sdk/pull/5533) Migrate the `x/supply` module to use Protocol Buffers for state - serialization instead of Amino. - _ The `internal` sub-package has been removed in order to expose the types proto file. - _ The `x/supply` module now accepts a `Codec` interface which extends the `codec.Marshaler` interface by - requiring a concrete codec to know how to serialize `SupplyI` types. \* The `SupplyI` interface has been modified to no longer return `SupplyI` on methods. Instead the - concrete type's receiver should modify the type. + serialization instead of Amino. + _ The `internal` sub-package has been removed in order to expose the types proto file. + _ The `x/supply` module now accepts a `Codec` interface which extends the `codec.Marshaler` interface by + requiring a concrete codec to know how to serialize `SupplyI` types. \* The `SupplyI` interface has been modified to no longer return `SupplyI` on methods. Instead the + concrete type's receiver should modify the type. * (x/upgrade) [#5659](https://github.com/cosmos/cosmos-sdk/pull/5659) Migrate the `x/upgrade` module to use Protocol - Buffers for state serialization instead of Amino. - _ The `internal` sub-package has been removed in order to expose the types proto file. - _ The `x/upgrade` module now accepts a `codec.Marshaler` interface. + Buffers for state serialization instead of Amino. + _ The `internal` sub-package has been removed in order to expose the types proto file. + _ The `x/upgrade` module now accepts a `codec.Marshaler` interface. ### Features @@ -1626,7 +1603,7 @@ sure you are aware of any relevant breaking changes. * **General** * (crypto/multisig) [#6241](https://github.com/cosmos/cosmos-sdk/pull/6241) Add Multisig type directly to the repo. Previously this was in tendermint. * (codec/types) [#8106](https://github.com/cosmos/cosmos-sdk/pull/8106) Adding `NewAnyWithCustomTypeURL` to correctly - marshal Messages in TxBuilder. + marshal Messages in TxBuilder. * (tests) [#6489](https://github.com/cosmos/cosmos-sdk/pull/6489) Introduce package `testutil`, new in-process testing network framework for use in integration and unit tests. * (tx) Add new auth/tx gRPC & gRPC-Gateway endpoints for basic querying & broadcasting support * [#7842](https://github.com/cosmos/cosmos-sdk/pull/7842) Add TxsByEvent gRPC endpoint @@ -1640,8 +1617,8 @@ sure you are aware of any relevant breaking changes. * **Modules** * (modules) [#5921](https://github.com/cosmos/cosmos-sdk/issues/5921) Introduction of Query gRPC service definitions along with REST annotations for gRPC Gateway for each module * (modules) [#7540](https://github.com/cosmos/cosmos-sdk/issues/7540) Protobuf service definitions can now be used for - packing `Msg`s in transactions as defined in [ADR 031](./docs/architecture/adr-031-msg-service.md). All modules now - define a `Msg` protobuf service. + packing `Msg`s in transactions as defined in [ADR 031](./docs/architecture/adr-031-msg-service.md). All modules now + define a `Msg` protobuf service. * (x/auth/vesting) [#7209](https://github.com/cosmos/cosmos-sdk/pull/7209) Create new `MsgCreateVestingAccount` message type along with CLI handler that allows for the creation of delayed and continuous vesting types. * (x/capability) [#5828](https://github.com/cosmos/cosmos-sdk/pull/5828) Capability module integration as outlined in [ADR 3 - Dynamic Capability Store](https://github.com/cosmos/tree/master/docs/architecture/adr-003-dynamic-capability-store.md). * (x/crisis) `x/crisis` has a new function: `AddModuleInitFlags`, which will register optional crisis module flags for the start command. @@ -1668,9 +1645,9 @@ sure you are aware of any relevant breaking changes. * (export) [#6510](https://github.com/cosmos/cosmos-sdk/pull/6510/) Field TimeIotaMs now is included in genesis file while exporting. * (rest) [#5906](https://github.com/cosmos/cosmos-sdk/pull/5906) Fix an issue that make some REST calls panic when sending invalid or incomplete requests. * (crypto) [#7966](https://github.com/cosmos/cosmos-sdk/issues/7966) `Bip44Params` `String()` function now correctly - returns the absolute HD path by adding the `m/` prefix. + returns the absolute HD path by adding the `m/` prefix. * (crypto/keyring) [#5844](https://github.com/cosmos/cosmos-sdk/pull/5844) `Keyring.Sign()` methods no longer decode amino signatures when method receivers - are offline/multisig keys. + are offline/multisig keys. * (store) [#7415](https://github.com/cosmos/cosmos-sdk/pull/7415) Allow new stores to be registered during on-chain upgrades. * **Modules** _ (modules) [#5569](https://github.com/cosmos/cosmos-sdk/issues/5569) `InitGenesis`, for the relevant modules, now ensures module accounts exist. @@ -1695,16 +1672,16 @@ sure you are aware of any relevant breaking changes. * (baseapp) [#6186](https://github.com/cosmos/cosmos-sdk/issues/6186) Support emitting events during `AnteHandler` execution. * (baseapp) [#6053](https://github.com/cosmos/cosmos-sdk/pull/6053) Customizable panic recovery handling added for `app.runTx()` method (as proposed in the [ADR 22](https://github.com/cosmos/cosmos-sdk/blob/master/docs/architecture/adr-022-custom-panic-handling.md)). Adds ability for developers to register custom panic handlers extending standard ones. * (client) [#5810](https://github.com/cosmos/cosmos-sdk/pull/5810) Added a new `--offline` flag that allows commands to be executed without an - internet connection. Previously, `--generate-only` served this purpose in addition to only allowing txs to be generated. Now, `--generate-only` solely - allows txs to be generated without being broadcasted and disallows Keybase use and `--offline` allows the use of Keybase but does not allow any - functionality that requires an online connection. + internet connection. Previously, `--generate-only` served this purpose in addition to only allowing txs to be generated. Now, `--generate-only` solely + allows txs to be generated without being broadcasted and disallows Keybase use and `--offline` allows the use of Keybase but does not allow any + functionality that requires an online connection. * (cli) [#7764](https://github.com/cosmos/cosmos-sdk/pull/7764) Update x/banking and x/crisis InitChain to improve node startup time * (client) [#5856](https://github.com/cosmos/cosmos-sdk/pull/5856) Added the possibility to set `--offline` flag with config command. * (client) [#5895](https://github.com/cosmos/cosmos-sdk/issues/5895) show config options in the config command's help screen. * (client/keys) [#8043](https://github.com/cosmos/cosmos-sdk/pull/8043) Add support for export of unarmored private key * (client/tx) [#7801](https://github.com/cosmos/cosmos-sdk/pull/7801) Update sign-batch multisig to work online * (x/genutil) [#8099](https://github.com/cosmos/cosmos-sdk/pull/8099) `init` now supports a `--recover` flag to recover - the private validator key from a given mnemonic + the private validator key from a given mnemonic * **Modules** * (x/auth) [#5702](https://github.com/cosmos/cosmos-sdk/pull/5702) Add parameter querying support for `x/auth`. * (x/auth/ante) [#6040](https://github.com/cosmos/cosmos-sdk/pull/6040) `AccountKeeper` interface used for `NewAnteHandler` and handler's decorators to add support of using custom `AccountKeeper` implementations. @@ -1715,16 +1692,16 @@ sure you are aware of any relevant breaking changes. * (x/staking) [#6059](https://github.com/cosmos/cosmos-sdk/pull/6059) Updated `HistoricalEntries` parameter default to 100. * (x/staking) [#5584](https://github.com/cosmos/cosmos-sdk/pull/5584) Add util function `ToTmValidator` that converts a `staking.Validator` type to `*tmtypes.Validator`. * (x/staking) [#6163](https://github.com/cosmos/cosmos-sdk/pull/6163) CLI and REST call to unbonding delegations and delegations now accept - pagination. + pagination. * (x/staking) [#8178](https://github.com/cosmos/cosmos-sdk/pull/8178) Update default historical header number for stargate * **General** * (crypto) [#7987](https://github.com/cosmos/cosmos-sdk/pull/7987) Fix the inconsistency of CryptoCdc, only use - `codec/legacy.Cdc`. + `codec/legacy.Cdc`. * (logging) [#8072](https://github.com/cosmos/cosmos-sdk/pull/8072) Refactor logging: - _ Use [zerolog](https://github.com/rs/zerolog) over Tendermint's go-kit logging wrapper. - _ Introduce Tendermint's `--log_format=plain|json` flag. Using format `json` allows for emitting structured JSON - logs which can be consumed by an external logging facility (e.g. Loggly). Both formats log to STDERR. \* The existing `--log_level` flag and it's default value now solely relates to the global logging - level (e.g. `info`, `debug`, etc...) instead of `:`. + _ Use [zerolog](https://github.com/rs/zerolog) over Tendermint's go-kit logging wrapper. + _ Introduce Tendermint's `--log_format=plain|json` flag. Using format `json` allows for emitting structured JSON + logs which can be consumed by an external logging facility (e.g. Loggly). Both formats log to STDERR. \* The existing `--log_level` flag and it's default value now solely relates to the global logging + level (e.g. `info`, `debug`, etc...) instead of `:`. * (rest) [#7649](https://github.com/cosmos/cosmos-sdk/pull/7649) Return an unsigned tx in legacy GET /tx endpoint when signature conversion fails * (simulation) [#6002](https://github.com/cosmos/cosmos-sdk/pull/6002) Add randomized consensus params into simulation. * (store) [#6481](https://github.com/cosmos/cosmos-sdk/pull/6481) Move `SimpleProofsFromMap` from Tendermint into the SDK. @@ -1746,7 +1723,7 @@ sure you are aware of any relevant breaking changes. * (types) [#6195](https://github.com/cosmos/cosmos-sdk/pull/6195) Add codespace to broadcast(sync/async) response. * (types) \#6897 Add KV type from tendermint to `types` directory. * (version) [#7848](https://github.com/cosmos/cosmos-sdk/pull/7848) [#7941](https://github.com/cosmos/cosmos-sdk/pull/7941) - `version --long` output now shows the list of build dependencies and replaced build dependencies. + `version --long` output now shows the list of build dependencies and replaced build dependencies. ## [v0.39.1](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.39.1) - 2020-08-11 @@ -1873,11 +1850,11 @@ sure you are aware of any relevant breaking changes. * (baseapp/types) [#5421](https://github.com/cosmos/cosmos-sdk/pull/5421) The `Error` interface (`types/errors.go`) has been removed in favor of the concrete type defined in `types/errors/` which implements the standard `error` interface. * As a result, the `Handler` and `Querier` implementations now return a standard `error`. - Within `BaseApp`, `runTx` now returns a `(GasInfo, *Result, error)`tuple and`runMsgs`returns a `(_Result, error)`tuple. A reference to a`Result`is now used to indicate success whereas an error signals an invalid message or failed message execution. As a result, the fields`Code`, `Codespace`, `GasWanted`, and `GasUsed`have been removed the`Result`type. The latter two fields are now found in the`GasInfo` type which is always returned regardless of execution outcome. - _ Note to developers: Since all handlers and queriers must now return a standard `error`, the `types/errors/` - package contains all the relevant and pre-registered errors that you typically work with. A typical - error returned will look like `sdkerrors.Wrap(sdkerrors.ErrUnknownRequest, "...")`. You can retrieve - relevant ABCI information from the error via `ABCIInfo`. + Within `BaseApp`, `runTx` now returns a `(GasInfo, *Result, error)`tuple and`runMsgs`returns a `(_Result, error)`tuple. A reference to a`Result`is now used to indicate success whereas an error signals an invalid message or failed message execution. As a result, the fields`Code`, `Codespace`, `GasWanted`, and `GasUsed`have been removed the`Result`type. The latter two fields are now found in the`GasInfo` type which is always returned regardless of execution outcome. + _ Note to developers: Since all handlers and queriers must now return a standard `error`, the `types/errors/` + package contains all the relevant and pre-registered errors that you typically work with. A typical + error returned will look like `sdkerrors.Wrap(sdkerrors.ErrUnknownRequest, "...")`. You can retrieve + relevant ABCI information from the error via `ABCIInfo`. * (client) [#5442](https://github.com/cosmos/cosmos-sdk/pull/5442) Remove client/alias.go as it's not necessary and components can be imported directly from the packages. * (store) [#4748](https://github.com/cosmos/cosmos-sdk/pull/4748) The `CommitMultiStore` interface @@ -1893,14 +1870,14 @@ sure you are aware of any relevant breaking changes. * (x/auth) [#5006](https://github.com/cosmos/cosmos-sdk/pull/5006) Modular `AnteHandler` via composable decorators: * The `AnteHandler` interface now returns `(newCtx Context, err error)` instead of `(newCtx Context, result sdk.Result, abort bool)` * The `NewAnteHandler` function returns an `AnteHandler` function that returns the new `AnteHandler` - interface and has been moved into the `auth/ante` directory. + interface and has been moved into the `auth/ante` directory. * `ValidateSigCount`, `ValidateMemo`, `ProcessPubKey`, `EnsureSufficientMempoolFee`, and `GetSignBytes` - have all been removed as public functions. + have all been removed as public functions. * Invalid Signatures may return `InvalidPubKey` instead of `Unauthorized` error, since the transaction - will first hit `SetPubKeyDecorator` before the `SigVerificationDecorator` runs. + will first hit `SetPubKeyDecorator` before the `SigVerificationDecorator` runs. * `StdTx#GetSignatures` will return an array of just signature byte slices `[][]byte` instead of - returning an array of `StdSignature` structs. To replicate the old behavior, use the public field - `StdTx.Signatures` to get back the array of StdSignatures `[]StdSignature`. + returning an array of `StdSignature` structs. To replicate the old behavior, use the public field + `StdTx.Signatures` to get back the array of StdSignatures `[]StdSignature`. * (modules) [#5299](https://github.com/cosmos/cosmos-sdk/pull/5299) `HandleDoubleSign` along with params `MaxEvidenceAge` and `DoubleSignJailEndTime` have moved from the `x/slashing` module to the `x/evidence` module. * (keys) [#4941](https://github.com/cosmos/cosmos-sdk/issues/4941) Keybase concrete types constructors such as `NewKeyBaseFromDir` and `NewInMemory` now accept optional parameters of type `KeybaseOption`. These optional parameters are also added on the keys sub-commands functions, which are now public, and allows @@ -1953,7 +1930,7 @@ sure you are aware of any relevant breaking changes. _ `os`: use OS default credentials storage (default). _ `file`: use encrypted file-based store. _ `kwallet`: use [KDE Wallet](https://utils.kde.org/projects/kwalletmanager/) service. - _ `pass`: use the [pass](https://www.passwordstore.org/) command line password manager. \* `test`: use password-less key store. *For testing purposes only. Use it at your own risk.* + _ `pass`: use the [pass](https://www.passwordstore.org/) command line password manager. \* `test`: use password-less key store. _For testing purposes only. Use it at your own risk._ * (keys) [#5097](https://github.com/cosmos/cosmos-sdk/pull/5097) New `keys migrate` command to assist users migrate their keys to the new keyring. * (keys) [#5366](https://github.com/cosmos/cosmos-sdk/pull/5366) `keys list` now accepts a `--list-names` option to list key names only, whilst the `keys delete` @@ -1967,26 +1944,26 @@ sure you are aware of any relevant breaking changes. * (x/auth/ante) [#5196](https://github.com/cosmos/cosmos-sdk/pull/5196) AnteDecorators have been updated to avoid unnecessary checks when `ctx.IsReCheckTx() == true` * (x/auth) [#5006](https://github.com/cosmos/cosmos-sdk/pull/5006) Modular `AnteHandler` via composable decorators: * The `AnteDecorator` interface has been introduced to allow users to implement modular `AnteHandler` - functionality that can be composed together to create a single `AnteHandler` rather than implementing - a custom `AnteHandler` completely from scratch, where each `AnteDecorator` allows for custom behavior in - tightly defined and logically isolated manner. These custom `AnteDecorator` can then be chained together - with default `AnteDecorator` or third-party `AnteDecorator` to create a modularized `AnteHandler` - which will run each `AnteDecorator` in the order specified in `ChainAnteDecorators`. For details - on the new architecture, refer to the [ADR](docs/architecture/adr-010-modular-antehandler.md). + functionality that can be composed together to create a single `AnteHandler` rather than implementing + a custom `AnteHandler` completely from scratch, where each `AnteDecorator` allows for custom behavior in + tightly defined and logically isolated manner. These custom `AnteDecorator` can then be chained together + with default `AnteDecorator` or third-party `AnteDecorator` to create a modularized `AnteHandler` + which will run each `AnteDecorator` in the order specified in `ChainAnteDecorators`. For details + on the new architecture, refer to the [ADR](docs/architecture/adr-010-modular-antehandler.md). * `ChainAnteDecorators` function has been introduced to take in a list of `AnteDecorators` and chain - them in sequence and return a single `AnteHandler`: - _ `SetUpContextDecorator`: Sets `GasMeter` in context and creates defer clause to recover from any - `OutOfGas` panics in future AnteDecorators and return `OutOfGas` error to `BaseApp`. It MUST be the - first `AnteDecorator` in the chain for any application that uses gas (or another one that sets the gas meter). - _ `ValidateBasicDecorator`: Calls tx.ValidateBasic and returns any non-nil error. - _ `ValidateMemoDecorator`: Validates tx memo with application parameters and returns any non-nil error. - _ `ConsumeGasTxSizeDecorator`: Consumes gas proportional to the tx size based on application parameters. - _ `MempoolFeeDecorator`: Checks if fee is above local mempool `minFee` parameter during `CheckTx`. - _ `DeductFeeDecorator`: Deducts the `FeeAmount` from first signer of the transaction. - _ `SetPubKeyDecorator`: Sets pubkey of account in any account that does not already have pubkey saved in state machine. - _ `SigGasConsumeDecorator`: Consume parameter-defined amount of gas for each signature. - _ `SigVerificationDecorator`: Verify each signature is valid, return if there is an error. - _ `ValidateSigCountDecorator`: Validate the number of signatures in tx based on app-parameters. \* `IncrementSequenceDecorator`: Increments the account sequence for each signer to prevent replay attacks. + them in sequence and return a single `AnteHandler`: + _ `SetUpContextDecorator`: Sets `GasMeter` in context and creates defer clause to recover from any + `OutOfGas` panics in future AnteDecorators and return `OutOfGas` error to `BaseApp`. It MUST be the + first `AnteDecorator` in the chain for any application that uses gas (or another one that sets the gas meter). + _ `ValidateBasicDecorator`: Calls tx.ValidateBasic and returns any non-nil error. + _ `ValidateMemoDecorator`: Validates tx memo with application parameters and returns any non-nil error. + _ `ConsumeGasTxSizeDecorator`: Consumes gas proportional to the tx size based on application parameters. + _ `MempoolFeeDecorator`: Checks if fee is above local mempool `minFee` parameter during `CheckTx`. + _ `DeductFeeDecorator`: Deducts the `FeeAmount` from first signer of the transaction. + _ `SetPubKeyDecorator`: Sets pubkey of account in any account that does not already have pubkey saved in state machine. + _ `SigGasConsumeDecorator`: Consume parameter-defined amount of gas for each signature. + _ `SigVerificationDecorator`: Verify each signature is valid, return if there is an error. + _ `ValidateSigCountDecorator`: Validate the number of signatures in tx based on app-parameters. \* `IncrementSequenceDecorator`: Increments the account sequence for each signer to prevent replay attacks. * (cli) [#5223](https://github.com/cosmos/cosmos-sdk/issues/5223) Cosmos Ledger App v2.0.0 is now supported. The changes are backwards compatible and App v1.5.x is still supported. * (x/staking) [#5380](https://github.com/cosmos/cosmos-sdk/pull/5380) Introduced ability to store historical info entries in staking keeper, allows applications to introspect specified number of past headers and validator sets * Introduces new parameter `HistoricalEntries` which allows applications to determine how many recent historical info entries they want to persist in store. Default value is 0. @@ -2042,9 +2019,9 @@ sure you are aware of any relevant breaking changes. * Add `RegisterStoreDecoders` to the `SimulationManager` for decoding each module's types * Add `GenerateGenesisStates` to the `SimulationManager` to generate a randomized `GenState` for each module * Add `RandomizedParams` to the `SimulationManager` that registers each modules' parameters in order to - simulate `ParamChangeProposal`s' `Content`s + simulate `ParamChangeProposal`s' `Content`s * Add `WeightedOperations` to the `SimulationManager` that define simulation operations (modules' `Msg`s) with their - respective weights (i.e chance of being simulated). + respective weights (i.e chance of being simulated). * Add `ProposalContents` to the `SimulationManager` to register each module's governance proposal `Content`s. * (simulation) [#4893](https://github.com/cosmos/cosmos-sdk/issues/4893) Change `SimApp` keepers to be public and add getter functions for keys and codec * (simulation) [#4906](https://github.com/cosmos/cosmos-sdk/issues/4906) Add simulation `Config` struct that wraps simulation flags @@ -2221,7 +2198,7 @@ sure you are aware of any relevant breaking changes. * (baseapp) [#4903](https://github.com/cosmos/cosmos-sdk/issues/4903) Various height query fixes: * Move height with proof check from `CLIContext` to `BaseApp` as the height - can automatically be injected there. + can automatically be injected there. * Update `handleQueryStore` to resemble `handleQueryCustom` * (simulation) [#4912](https://github.com/cosmos/cosmos-sdk/issues/4912) Fix SimApp ModuleAccountAddrs to properly return black listed addresses for bank keeper initialization. @@ -2260,7 +2237,7 @@ sure you are aware of any relevant breaking changes. * [#3628](https://github.com/cosmos/cosmos-sdk/issues/3628) Replaced governance's burn and deposit accounts for a `ModuleAccount` * Added a `ModuleAccount` for the distribution module * Added a `ModuleAccount` for the mint module - [#4472](https://github.com/cosmos/cosmos-sdk/issues/4472) validation for crisis genesis + [#4472](https://github.com/cosmos/cosmos-sdk/issues/4472) validation for crisis genesis * [#3985](https://github.com/cosmos/cosmos-sdk/issues/3985) `ValidatorPowerRank` uses potential consensus power instead of tendermint power * [#4104](https://github.com/cosmos/cosmos-sdk/issues/4104) Gaia has been moved to its own repository: https://github.com/cosmos/gaia * [#4104](https://github.com/cosmos/cosmos-sdk/issues/4104) Rename gaiad.toml to app.toml. The internal contents of the application @@ -2283,10 +2260,10 @@ sure you are aware of any relevant breaking changes. * Update `Context` to use new `EventManager` * (Begin|End)Blocker no longer return tags, but rather uses new `EventManager` * Message handlers no longer return tags, but rather uses new `EventManager` - Any component (e.g. BeginBlocker, message handler, etc...) wishing to emit an event must do so - through `ctx.EventManger().EmitEvent(s)`. - To reset or wipe emitted events: `ctx = ctx.WithEventManager(sdk.NewEventManager())` - To get all emitted events: `events := ctx.EventManager().Events()` + Any component (e.g. BeginBlocker, message handler, etc...) wishing to emit an event must do so + through `ctx.EventManger().EmitEvent(s)`. + To reset or wipe emitted events: `ctx = ctx.WithEventManager(sdk.NewEventManager())` + To get all emitted events: `events := ctx.EventManager().Events()` * [#4437](https://github.com/cosmos/cosmos-sdk/issues/4437) Replace governance module store keys to use `[]byte` instead of `string`. * [#4451](https://github.com/cosmos/cosmos-sdk/issues/4451) Improve modularization of clients and modules: * Module directory structure improved and standardized @@ -2307,7 +2284,7 @@ sure you are aware of any relevant breaking changes. * GetAccountDecoder() * CLIContext.WithAccountDecoder() * CLIContext.WithAccountStore() - x/auth.AccountDecoder is unnecessary and consequently removed. + x/auth.AccountDecoder is unnecessary and consequently removed. * [#4602](https://github.com/cosmos/cosmos-sdk/issues/4602) client/input.{Buffer,Override}Stdin() functions are removed. Thanks to cobra's new release they are now redundant. * [#4633](https://github.com/cosmos/cosmos-sdk/issues/4633) Update old Tx search by tags APIs to use new Events nomenclature. @@ -2755,25 +2732,25 @@ BREAKING CHANGES * Gaia * [#3787](https://github.com/cosmos/cosmos-sdk/pull/3787) Fork the `x/bank` module into the Gaia application with only a - modified message handler, where the modified message handler behaves the same as - the standard `x/bank` message handler except for `MsgMultiSend` that must burn - exactly 9 atoms and transfer 1 atom, and `MsgSend` is disabled. + modified message handler, where the modified message handler behaves the same as + the standard `x/bank` message handler except for `MsgMultiSend` that must burn + exactly 9 atoms and transfer 1 atom, and `MsgSend` is disabled. * [#3789](https://github.com/cosmos/cosmos-sdk/pull/3789) Update validator creation flow: - _ Remove `NewMsgCreateValidatorOnBehalfOf` and corresponding business logic - _ Ensure the validator address equals the delegator address during - `MsgCreateValidator#ValidateBasic` + _ Remove `NewMsgCreateValidatorOnBehalfOf` and corresponding business logic + _ Ensure the validator address equals the delegator address during + `MsgCreateValidator#ValidateBasic` * SDK * [#3750](https://github.com/cosmos/cosmos-sdk/issues/3750) Track outstanding rewards per-validator instead of globally, - and fix the main simulation issue, which was that slashes of - re-delegations to a validator were not correctly accounted for - in fee distribution when the redelegation in question had itself - been slashed (from a fault committed by a different validator) - in the same BeginBlock. Outstanding rewards are now available - on a per-validator basis in REST. + and fix the main simulation issue, which was that slashes of + re-delegations to a validator were not correctly accounted for + in fee distribution when the redelegation in question had itself + been slashed (from a fault committed by a different validator) + in the same BeginBlock. Outstanding rewards are now available + on a per-validator basis in REST. * [#3669](https://github.com/cosmos/cosmos-sdk/pull/3669) Ensure consistency in message naming, codec registration, and JSON - tags. + tags. * [#3788](https://github.com/cosmos/cosmos-sdk/pull/3788) Change order of operations for greater accuracy when calculating delegation share token value * [#3788](https://github.com/cosmos/cosmos-sdk/pull/3788) DecCoins.Cap -> DecCoins.Intersect * [#3666](https://github.com/cosmos/cosmos-sdk/pull/3666) Improve coins denom validation. @@ -2786,7 +2763,7 @@ FEATURES * SDK * [#3719](https://github.com/cosmos/cosmos-sdk/issues/3719) DBBackend can now be set at compile time. - Defaults: goleveldb. Supported: cleveldb. + Defaults: goleveldb. Supported: cleveldb. IMPROVEMENTS @@ -2799,20 +2776,20 @@ IMPROVEMENTS * [#3653](https://github.com/cosmos/cosmos-sdk/pull/3653) Prompt user confirmation prior to signing and broadcasting a transaction. * [#3670](https://github.com/cosmos/cosmos-sdk/pull/3670) CLI support for showing bech32 addresses in Ledger devices * [#3711](https://github.com/cosmos/cosmos-sdk/pull/3711) Update `tx sign` to use `--from` instead of the deprecated `--name` - CLI flag. + CLI flag. * [#3738](https://github.com/cosmos/cosmos-sdk/pull/3738) Improve multisig UX: * `gaiacli keys show -o json` now includes constituent pubkeys, respective weights and threshold * `gaiacli keys show --show-multisig` now displays constituent pubkeys, respective weights and threshold * `gaiacli tx sign --validate-signatures` now displays multisig signers with their respective weights * [#3730](https://github.com/cosmos/cosmos-sdk/issues/3730) Improve workflow for - `gaiad gentx` with offline public keys, by outputting stdtx file that needs to be signed. + `gaiad gentx` with offline public keys, by outputting stdtx file that needs to be signed. * [#3761](https://github.com/cosmos/cosmos-sdk/issues/3761) Querying account related information using custom querier in auth module * SDK * [#3753](https://github.com/cosmos/cosmos-sdk/issues/3753) Remove no-longer-used governance penalty parameter * [#3679](https://github.com/cosmos/cosmos-sdk/issues/3679) Consistent operators across Coins, DecCoins, Int, Dec - replaced: Minus->Sub Plus->Add Div->Quo + replaced: Minus->Sub Plus->Add Div->Quo * [#3665](https://github.com/cosmos/cosmos-sdk/pull/3665) Overhaul sdk.Uint type in preparation for Coins Int -> Uint migration. * [#3691](https://github.com/cosmos/cosmos-sdk/issues/3691) Cleanup error messages * [#3456](https://github.com/cosmos/cosmos-sdk/issues/3456) Integrate in the Int.ToDec() convenience function @@ -2838,18 +2815,18 @@ BUG FIXES * SDK * [#3728](https://github.com/cosmos/cosmos-sdk/issues/3728) Truncate decimal multiplication & division in distribution to ensure - no more than the collected fees / inflation are distributed + no more than the collected fees / inflation are distributed * [#3727](https://github.com/cosmos/cosmos-sdk/issues/3727) Return on zero-length (including []byte{}) PrefixEndBytes() calls * [#3559](https://github.com/cosmos/cosmos-sdk/issues/3559) fix occasional failing due to non-determinism in lcd test TestBonding - where validator is unexpectedly slashed throwing off test calculations + where validator is unexpectedly slashed throwing off test calculations * [#3411](https://github.com/cosmos/cosmos-sdk/pull/3411) Include the `RequestInitChain.Time` in the block header init during - `InitChain`. + `InitChain`. * [#3717](https://github.com/cosmos/cosmos-sdk/pull/3717) Update the vesting specification and implementation to cap deduction from - `DelegatedVesting` by at most `DelegatedVesting`. This accounts for the case where - the undelegation amount may exceed the original delegation amount due to - truncation of undelegation tokens. + `DelegatedVesting` by at most `DelegatedVesting`. This accounts for the case where + the undelegation amount may exceed the original delegation amount due to + truncation of undelegation tokens. * [#3717](https://github.com/cosmos/cosmos-sdk/pull/3717) Ignore unknown proposers in allocating rewards for proposers, in case - unbonding period was just 1 block and proposer was already deleted. + unbonding period was just 1 block and proposer was already deleted. * [#3726](https://github.com/cosmos/cosmos-sdk/pull/3724) Cap(clip) reward to remaining coins in AllocateTokens. ## 0.32.0 @@ -2876,9 +2853,9 @@ IMPROVEMENTS * [#3614](https://github.com/cosmos/cosmos-sdk/pull/3614) Add coin denom length checks to the coins constructors. * [#3621](https://github.com/cosmos/cosmos-sdk/issues/3621) remove many inter-module dependancies * [#3601](https://github.com/cosmos/cosmos-sdk/pull/3601) JSON-stringify the ABCI log response which includes the log and message - index. + index. * [#3604](https://github.com/cosmos/cosmos-sdk/pull/3604) Improve SDK funds related error messages and allow for unicode in - JSON ABCI log. + JSON ABCI log. * [#3620](https://github.com/cosmos/cosmos-sdk/pull/3620) Version command shows build tags * [#3638](https://github.com/cosmos/cosmos-sdk/pull/3638) Add Bcrypt benchmarks & justification of security parameter choice * [#3648](https://github.com/cosmos/cosmos-sdk/pull/3648) Add JSON struct tags to vesting accounts. @@ -2907,7 +2884,7 @@ IMPROVEMENTS * SDK * [#3604](https://github.com/cosmos/cosmos-sdk/pulls/3604) Improve SDK funds related error messages and allow for unicode in - JSON ABCI log. + JSON ABCI log. * Tendermint * [#3563](https://github.com/cosmos/cosmos-sdk/3563) Update to Tendermint version `0.30.0-rc0` @@ -2918,12 +2895,12 @@ BUG FIXES * [#3585] Fix setting the tx hash in `NewResponseFormatBroadcastTxCommit`. * [#3585] Return an empty `TxResponse` when Tendermint returns an empty - `ResultBroadcastTx`. + `ResultBroadcastTx`. * SDK * [#3582](https://github.com/cosmos/cosmos-sdk/pull/3582) Running `make test_unit` was failing due to a missing tag * [#3617](https://github.com/cosmos/cosmos-sdk/pull/3582) Fix fee comparison when the required fees does not contain any denom - present in the tx fees. + present in the tx fees. ## 0.31.0 @@ -2932,7 +2909,7 @@ BREAKING CHANGES * Gaia REST API (`gaiacli advanced rest-server`) * [#3284](https://github.com/cosmos/cosmos-sdk/issues/3284) Rename the `name` - field to `from` in the `base_req` body. + field to `from` in the `base_req` body. * [#3485](https://github.com/cosmos/cosmos-sdk/pull/3485) Error responses are now JSON objects. * [#3477][distribution] endpoint changed "all_delegation_rewards" -> "delegator_total_rewards" @@ -2946,7 +2923,7 @@ BREAKING CHANGES * [#3451](https://github.com/cosmos/cosmos-sdk/pull/3451) `gaiacli` now returns transactions in plain text including tags. * [#3497](https://github.com/cosmos/cosmos-sdk/issues/3497) `gaiad init` now takes moniker as required arguments, not as parameter. * [#3501](https://github.com/cosmos/cosmos-sdk/issues/3501) Change validator - address Bech32 encoding to consensus address in `tendermint-validator-set`. + address Bech32 encoding to consensus address in `tendermint-validator-set`. * Gaia @@ -2974,10 +2951,10 @@ FEATURES * Gaia CLI (`gaiacli`) * [#3429](https://github.com/cosmos/cosmos-sdk/issues/3429) Support querying - for all delegator distribution rewards. + for all delegator distribution rewards. * [#3449](https://github.com/cosmos/cosmos-sdk/issues/3449) Proof verification now works with absence proofs * [#3484](https://github.com/cosmos/cosmos-sdk/issues/3484) Add support - vesting accounts to the add-genesis-account command. + vesting accounts to the add-genesis-account command. * Gaia @@ -2988,7 +2965,7 @@ FEATURES * [#3270](https://github.com/cosmos/cosmos-sdk/issues/3270) [x/staking] limit number of ongoing unbonding delegations /redelegations per pair/trio * [#3477][distribution] new query endpoint "delegator_validators" * [#3514](https://github.com/cosmos/cosmos-sdk/pull/3514) Provided a lazy loading implementation of Keybase that locks the underlying - storage only for the time needed to perform the required operation. Also added Keybase reference to TxBuilder struct. + storage only for the time needed to perform the required operation. Also added Keybase reference to TxBuilder struct. * [types] [#2580](https://github.com/cosmos/cosmos-sdk/issues/2580) Addresses now Bech32 empty addresses to an empty string IMPROVEMENTS @@ -2996,11 +2973,11 @@ IMPROVEMENTS * Gaia REST API * [#3284](https://github.com/cosmos/cosmos-sdk/issues/3284) Update Gaia Lite - REST service to support the following: - _ Automatic account number and sequence population when fields are omitted - _ Generate only functionality no longer requires access to a local Keybase \* `from` field in the `base_req` body can be a Keybase name or account address + REST service to support the following: + _ Automatic account number and sequence population when fields are omitted + _ Generate only functionality no longer requires access to a local Keybase \* `from` field in the `base_req` body can be a Keybase name or account address * [#3423](https://github.com/cosmos/cosmos-sdk/issues/3423) Allow simulation - (auto gas) to work with generate only. + (auto gas) to work with generate only. * [#3514](https://github.com/cosmos/cosmos-sdk/pull/3514) REST server calls to keybase does not lock the underlying storage anymore. * [#3523](https://github.com/cosmos/cosmos-sdk/pull/3523) Added `/tx/encode` endpoint to serialize a JSON tx to base64-encoded Amino. @@ -3009,17 +2986,17 @@ IMPROVEMENTS * [#3476](https://github.com/cosmos/cosmos-sdk/issues/3476) New `withdraw-all-rewards` command to withdraw all delegations rewards for delegators. * [#3497](https://github.com/cosmos/cosmos-sdk/issues/3497) `gaiad gentx` supports `--ip` and `--node-id` flags to override defaults. * [#3518](https://github.com/cosmos/cosmos-sdk/issues/3518) Fix flow in - `keys add` to show the mnemonic by default. + `keys add` to show the mnemonic by default. * [#3517](https://github.com/cosmos/cosmos-sdk/pull/3517) Increased test coverage * [#3523](https://github.com/cosmos/cosmos-sdk/pull/3523) Added `tx encode` command to serialize a JSON tx to base64-encoded Amino. * Gaia * [#3418](https://github.com/cosmos/cosmos-sdk/issues/3418) Add vesting account - genesis validation checks to `GaiaValidateGenesisState`. + genesis validation checks to `GaiaValidateGenesisState`. * [#3420](https://github.com/cosmos/cosmos-sdk/issues/3420) Added maximum length to governance proposal descriptions and titles * [#3256](https://github.com/cosmos/cosmos-sdk/issues/3256) Add gas consumption - for tx size in the ante handler. + for tx size in the ante handler. * [#3454](https://github.com/cosmos/cosmos-sdk/pull/3454) Add `--jail-whitelist` to `gaiad export` to enable testing of complex exports * [#3424](https://github.com/cosmos/cosmos-sdk/issues/3424) Allow generation of gentxs with empty memo field. * [#3507](https://github.com/cosmos/cosmos-sdk/issues/3507) General cleanup, removal of unnecessary struct fields, undelegation bugfix, and comment clarification in x/staking and x/slashing @@ -3030,10 +3007,10 @@ IMPROVEMENTS * [#3435](https://github.com/cosmos/cosmos-sdk/issues/3435) Test that store implementations do not allow nil values * [#2509](https://github.com/cosmos/cosmos-sdk/issues/2509) Sanitize all usage of Dec.RoundInt64() * [#556](https://github.com/cosmos/cosmos-sdk/issues/556) Increase `BaseApp` - test coverage. + test coverage. * [#3357](https://github.com/cosmos/cosmos-sdk/issues/3357) develop state-transitions.md for staking spec, missing states added to `state.md` * [#3552](https://github.com/cosmos/cosmos-sdk/pull/3552) Validate bit length when - deserializing `Int` types. + deserializing `Int` types. BUG FIXES @@ -3041,16 +3018,16 @@ BUG FIXES * [#3417](https://github.com/cosmos/cosmos-sdk/pull/3417) Fix `q slashing signing-info` panic by ensuring safety of user input and properly returning not found error * [#3345](https://github.com/cosmos/cosmos-sdk/issues/3345) Upgrade ledger-cosmos-go dependency to v0.9.3 to pull - https://github.com/ZondaX/ledger-cosmos-go/commit/ed9aa39ce8df31bad1448c72d3d226bf2cb1a8d1 in order to fix a derivation path issue that causes `gaiacli keys add --recover` - to malfunction. + https://github.com/ZondaX/ledger-cosmos-go/commit/ed9aa39ce8df31bad1448c72d3d226bf2cb1a8d1 in order to fix a derivation path issue that causes `gaiacli keys add --recover` + to malfunction. * [#3419](https://github.com/cosmos/cosmos-sdk/pull/3419) Fix `q distr slashes` panic * [#3453](https://github.com/cosmos/cosmos-sdk/pull/3453) The `rest-server` command didn't respect persistent flags such as `--chain-id` and `--trust-node` if they were - passed on the command line. + passed on the command line. * [#3441](https://github.com/cosmos/cosmos-sdk/pull/3431) Improved resource management and connection handling (ledger devices). Fixes issue with DER vs BER signatures. * Gaia * [#3486](https://github.com/cosmos/cosmos-sdk/pull/3486) Use AmountOf in - vesting accounts instead of zipping/aligning denominations. + vesting accounts instead of zipping/aligning denominations. ## 0.30.0 @@ -3082,7 +3059,7 @@ BREAKING CHANGES * https://github.com/cosmos/cosmos-sdk/issues/2838 - Move store keys to constants * [#3162](https://github.com/cosmos/cosmos-sdk/issues/3162) The `--gas` flag now takes `auto` instead of `simulate` - in order to trigger a simulation of the tx before the actual execution. + in order to trigger a simulation of the tx before the actual execution. * [#3285](https://github.com/cosmos/cosmos-sdk/pull/3285) New `gaiad tendermint version` to print libs versions * [#1894](https://github.com/cosmos/cosmos-sdk/pull/1894) `version` command now shows latest commit, vendor dir hash, and build machine info. * [#3249](https://github.com/cosmos/cosmos-sdk/issues/3249) `tendermint`'s `show-validator` and `show-address` `--json` flags removed in favor of `--output-format=json`. @@ -3091,14 +3068,14 @@ BREAKING CHANGES * [distribution] [#3359](https://github.com/cosmos/cosmos-sdk/issues/3359) Always round down when calculating rewards-to-be-withdrawn in F1 fee distribution * [#3336](https://github.com/cosmos/cosmos-sdk/issues/3336) Ensure all SDK - messages have their signature bytes contain canonical fields `value` and `type`. + messages have their signature bytes contain canonical fields `value` and `type`. * [#3333](https://github.com/cosmos/cosmos-sdk/issues/3333) - F1 storage efficiency improvements - automatic withdrawals when unbonded, historical reward reference counting * [staking] [#2513](https://github.com/cosmos/cosmos-sdk/issues/2513) Validator power type from Dec -> Int * [staking] [#3233](https://github.com/cosmos/cosmos-sdk/issues/3233) key and value now contain duplicate fields to simplify code * [#3064](https://github.com/cosmos/cosmos-sdk/issues/3064) Sanitize `sdk.Coin` denom. Coins denoms are now case insensitive, i.e. 100fooToken equals to 100FOOTOKEN. * [#3195](https://github.com/cosmos/cosmos-sdk/issues/3195) Allows custom configuration for syncable strategy * [#3242](https://github.com/cosmos/cosmos-sdk/issues/3242) Fix infinite gas - meter utilization during aborted ante handler executions. + meter utilization during aborted ante handler executions. * [x/distribution] [#3292](https://github.com/cosmos/cosmos-sdk/issues/3292) Enable or disable withdraw addresses with a parameter in the param store * [staking] [#2222](https://github.com/cosmos/cosmos-sdk/issues/2222) `/stake` -> `/staking` module rename * [staking] [#1268](https://github.com/cosmos/cosmos-sdk/issues/1268) `LooseTokens` -> `NotBondedTokens` @@ -3121,14 +3098,14 @@ FEATURES * [#3067](https://github.com/cosmos/cosmos-sdk/issues/3067) Add support for fees on transactions * [#3069](https://github.com/cosmos/cosmos-sdk/pull/3069) Add a custom memo on transactions * [#3027](https://github.com/cosmos/cosmos-sdk/issues/3027) Implement - `/gov/proposals/{proposalID}/proposer` to query for a proposal's proposer. + `/gov/proposals/{proposalID}/proposer` to query for a proposal's proposer. * Gaia CLI (`gaiacli`) * [#2399](https://github.com/cosmos/cosmos-sdk/issues/2399) Implement `params` command to query slashing parameters. * [#2730](https://github.com/cosmos/cosmos-sdk/issues/2730) Add tx search pagination parameter * [#3027](https://github.com/cosmos/cosmos-sdk/issues/3027) Implement - `query gov proposer [proposal-id]` to query for a proposal's proposer. + `query gov proposer [proposal-id]` to query for a proposal's proposer. * [#3198](https://github.com/cosmos/cosmos-sdk/issues/3198) New `keys add --multisig` flag to store multisig keys locally. * [#3198](https://github.com/cosmos/cosmos-sdk/issues/3198) New `multisign` command to generate multisig signatures. * [#3198](https://github.com/cosmos/cosmos-sdk/issues/3198) New `sign --multisig` flag to enable multisig mode. @@ -3140,7 +3117,7 @@ FEATURES * [#2182] [x/staking] Added querier for querying a single redelegation * [#3305](https://github.com/cosmos/cosmos-sdk/issues/3305) Add support for - vesting accounts at genesis. + vesting accounts at genesis. * [#3198](https://github.com/cosmos/cosmos-sdk/issues/3198) [x/auth] Add multisig transactions support * [#3198](https://github.com/cosmos/cosmos-sdk/issues/3198) `add-genesis-account` can take both account addresses and key names @@ -3149,7 +3126,7 @@ FEATURES * [#2926](https://github.com/cosmos/cosmos-sdk/issues/2926) Add TxEncoder to client TxBuilder. * [#2694](https://github.com/cosmos/cosmos-sdk/issues/2694) Vesting account implementation. * [#2996](https://github.com/cosmos/cosmos-sdk/issues/2996) Update the `AccountKeeper` to contain params used in the context of - the ante handler. + the ante handler. * [#3179](https://github.com/cosmos/cosmos-sdk/pull/3179) New CodeNoSignatures error code. * [#3319](https://github.com/cosmos/cosmos-sdk/issues/3319) [x/distribution] Queriers for all distribution state worth querying; distribution query commands * [#3356](https://github.com/cosmos/cosmos-sdk/issues/3356) [x/auth] bech32-ify accounts address in error message. @@ -3172,11 +3149,11 @@ IMPROVEMENTS * [#3172](https://github.com/cosmos/cosmos-sdk/pull/3172) Support minimum fees in a local testnet. * [#3250](https://github.com/cosmos/cosmos-sdk/pull/3250) Refactor integration tests and increase coverage * [#3248](https://github.com/cosmos/cosmos-sdk/issues/3248) Refactor tx fee - model: - _ Validators specify minimum gas prices instead of minimum fees - _ Clients may provide either fees or gas prices directly - _ The gas prices of a tx must meet a validator's minimum - _ `gaiad start` and `gaia.toml` take --minimum-gas-prices flag and minimum-gas-price config key respectively. + model: + _ Validators specify minimum gas prices instead of minimum fees + _ Clients may provide either fees or gas prices directly + _ The gas prices of a tx must meet a validator's minimum + _ `gaiad start` and `gaia.toml` take --minimum-gas-prices flag and minimum-gas-price config key respectively. * [#2859](https://github.com/cosmos/cosmos-sdk/issues/2859) Rename `TallyResult` in gov proposals to `FinalTallyResult` * [#3286](https://github.com/cosmos/cosmos-sdk/pull/3286) Fix `gaiad gentx` printout of account's addresses, i.e. user bech32 instead of hex. * [#3249](https://github.com/cosmos/cosmos-sdk/issues/3249) `--json` flag removed, users should use `--output=json` instead. @@ -3184,10 +3161,10 @@ IMPROVEMENTS * SDK * [#3137](https://github.com/cosmos/cosmos-sdk/pull/3137) Add tag documentation - for each module along with cleaning up a few existing tags in the governance, - slashing, and staking modules. + for each module along with cleaning up a few existing tags in the governance, + slashing, and staking modules. * [#3093](https://github.com/cosmos/cosmos-sdk/issues/3093) Ante handler does no longer read all accounts in one go when processing signatures as signature - verification may fail before last signature is checked. + verification may fail before last signature is checked. * [staking] [#1402](https://github.com/cosmos/cosmos-sdk/issues/1402) Add for multiple simultaneous redelegations or unbonding-delegations within an unbonding period * [staking] [#1268](https://github.com/cosmos/cosmos-sdk/issues/1268) staking spec rewrite @@ -3207,10 +3184,10 @@ BUG FIXES * [#3148](https://github.com/cosmos/cosmos-sdk/issues/3148) Fix `gaiad export` by adding a boolean to `NewGaiaApp` determining whether or not to load the latest version * [#3181](https://github.com/cosmos/cosmos-sdk/issues/3181) Correctly reset total accum update height and jailed-validator bond height / unbonding height on export-for-zero-height * [#3172](https://github.com/cosmos/cosmos-sdk/pull/3172) Fix parsing `gaiad.toml` - when it already exists. + when it already exists. * [#3223](https://github.com/cosmos/cosmos-sdk/issues/3223) Fix unset governance proposal queues when importing state from old chain * [#3187](https://github.com/cosmos/cosmos-sdk/issues/3187) Fix `gaiad export` - by resetting each validator's slashing period. + by resetting each validator's slashing period. ## 0.29.1 @@ -3256,12 +3233,12 @@ IMPROVEMENTS * Gaia REST API (`gaiacli advanced rest-server`) * [#2879](https://github.com/cosmos/cosmos-sdk/issues/2879), [#2880](https://github.com/cosmos/cosmos-sdk/issues/2880) Update deposit and vote endpoints to perform a direct txs query - when a given proposal is inactive and thus having votes and deposits removed - from state. + when a given proposal is inactive and thus having votes and deposits removed + from state. * Gaia CLI (`gaiacli`) * [#2879](https://github.com/cosmos/cosmos-sdk/issues/2879), [#2880](https://github.com/cosmos/cosmos-sdk/issues/2880) Update deposit and vote CLI commands to perform a direct txs query - when a given proposal is inactive and thus having votes and deposits removed - from state. + when a given proposal is inactive and thus having votes and deposits removed + from state. * Gaia * [#3021](https://github.com/cosmos/cosmos-sdk/pull/3021) Add `--gentx-dir` to `gaiad collect-gentxs` to specify a directory from which collect and load gentxs. Add `--output-document` to `gaiad init` to allow one to redirect output to file. @@ -3306,7 +3283,7 @@ IMPROVEMENTS * [#1277](https://github.com/cosmos/cosmos-sdk/issues/1277) Complete bank module specification * [#2963](https://github.com/cosmos/cosmos-sdk/issues/2963) Complete auth module specification * [#2914](https://github.com/cosmos/cosmos-sdk/issues/2914) No longer withdraw validator rewards on bond/unbond, but rather move - the rewards to the respective validator's pools. + the rewards to the respective validator's pools. BUG FIXES @@ -3362,12 +3339,12 @@ FEATURES * Gaia REST API (`gaiacli advanced rest-server`) * [gov] [#2479](https://github.com/cosmos/cosmos-sdk/issues/2479) Added governance parameter - query REST endpoints. + query REST endpoints. * Gaia CLI (`gaiacli`) * [gov][cli] [#2479](https://github.com/cosmos/cosmos-sdk/issues/2479) Added governance - parameter query commands. + parameter query commands. * [stake][cli] [#2027] Add CLI query command for getting all delegations to a specific validator. * [#2840](https://github.com/cosmos/cosmos-sdk/pull/2840) Standardize CLI exports from modules @@ -3375,7 +3352,7 @@ FEATURES * [app] [#2791](https://github.com/cosmos/cosmos-sdk/issues/2791) Support export at a specific height, with `gaiad export --height=HEIGHT`. * [x/gov] [#2479](https://github.com/cosmos/cosmos-sdk/issues/2479) Implemented querier - for getting governance parameters. + for getting governance parameters. * [app] [#2663](https://github.com/cosmos/cosmos-sdk/issues/2663) - Runtime-assertable invariants * [app] [#2791](https://github.com/cosmos/cosmos-sdk/issues/2791) Support export at a specific height, with `gaiad export --height=HEIGHT`. * [app] [#2812](https://github.com/cosmos/cosmos-sdk/issues/2812) Support export alterations to prepare for restarting at zero-height @@ -3503,7 +3480,7 @@ BUG FIXES ## 0.25.0 -*October 24th, 2018*. +_October 24th, 2018_. BREAKING CHANGES @@ -3523,8 +3500,8 @@ BREAKING CHANGES * [cli] [#2014](https://github.com/cosmos/cosmos-sdk/issues/2014) `gaiacli advanced` no longer exists - to access `ibc`, `rest-server`, and `validator-set` commands use `gaiacli ibc`, `gaiacli rest-server`, and `gaiacli tendermint`, respectively * [makefile] `get_vendor_deps` no longer updates lock file it just updates vendor directory. Use `update_vendor_deps` to update the lock file. [#2152](https://github.com/cosmos/cosmos-sdk/pull/2152) * [cli] [#2221](https://github.com/cosmos/cosmos-sdk/issues/2221) All commands that - utilize a validator's operator address must now use the new Bech32 prefix, - `cosmosvaloper`. + utilize a validator's operator address must now use the new Bech32 prefix, + `cosmosvaloper`. * [cli] [#2190](https://github.com/cosmos/cosmos-sdk/issues/2190) `gaiacli init --gen-txs` is now `gaiacli init --with-txs` to reduce confusion * [cli] [#2073](https://github.com/cosmos/cosmos-sdk/issues/2073) --from can now be either an address or a key name * [cli] [#1184](https://github.com/cosmos/cosmos-sdk/issues/1184) Subcommands reorganisation, see [#2390](https://github.com/cosmos/cosmos-sdk/pull/2390) for a comprehensive list of changes. @@ -3541,13 +3518,13 @@ BREAKING CHANGES * [x/stake] [#1877] Redelegations/unbonding-delegation from unbonding validator have reduced time * [x/slashing] [#1789](https://github.com/cosmos/cosmos-sdk/issues/1789) Slashing changes for Tendermint validator set offset (NextValSet) * [x/stake] [#2040](https://github.com/cosmos/cosmos-sdk/issues/2040) Validator - operator type has now changed to `sdk.ValAddress` + operator type has now changed to `sdk.ValAddress` * [x/stake] [#2221](https://github.com/cosmos/cosmos-sdk/issues/2221) New - Bech32 prefixes have been introduced for a validator's consensus address and - public key: `cosmosvalcons` and `cosmosvalconspub` respectively. Also, existing Bech32 prefixes have been - renamed for accounts and validator operators: - _ `cosmosaccaddr` / `cosmosaccpub` => `cosmos` / `cosmospub` - _ `cosmosvaladdr` / `cosmosvalpub` => `cosmosvaloper` / `cosmosvaloperpub` + Bech32 prefixes have been introduced for a validator's consensus address and + public key: `cosmosvalcons` and `cosmosvalconspub` respectively. Also, existing Bech32 prefixes have been + renamed for accounts and validator operators: + _ `cosmosaccaddr` / `cosmosaccpub` => `cosmos` / `cosmospub` + _ `cosmosvaladdr` / `cosmosvalpub` => `cosmosvaloper` / `cosmosvaloperpub` * [x/stake] [#1013] TendermintUpdates now uses transient store * [x/stake] [#2435](https://github.com/cosmos/cosmos-sdk/issues/2435) Remove empty bytes from the ValidatorPowerRank store key * [x/gov] [#2195](https://github.com/cosmos/cosmos-sdk/issues/2195) Governance uses BFT Time @@ -3647,9 +3624,9 @@ FEATURES * [cli] Cmds to query staking pool and params * [gov][cli] [#2062](https://github.com/cosmos/cosmos-sdk/issues/2062) added `--proposal` flag to `submit-proposal` that allows a JSON file containing a proposal to be passed in * [#2040](https://github.com/cosmos/cosmos-sdk/issues/2040) Add `--bech` to `gaiacli keys show` and respective REST endpoint to - provide desired Bech32 prefix encoding + provide desired Bech32 prefix encoding * [cli] [#2047](https://github.com/cosmos/cosmos-sdk/issues/2047) [#2306](https://github.com/cosmos/cosmos-sdk/pull/2306) Passing --gas=simulate triggers a simulation of the tx before the actual execution. - The gas estimate obtained via the simulation will be used as gas limit in the actual execution. + The gas estimate obtained via the simulation will be used as gas limit in the actual execution. * [cli] [#2047](https://github.com/cosmos/cosmos-sdk/issues/2047) The --gas-adjustment flag can be used to adjust the estimate obtained via the simulation triggered by --gas=simulate. * [cli] [#2110](https://github.com/cosmos/cosmos-sdk/issues/2110) Add --dry-run flag to perform a simulation of a transaction without broadcasting it. The --gas flag is ignored as gas would be automatically estimated. * [cli] [#2204](https://github.com/cosmos/cosmos-sdk/issues/2204) Support generating and broadcasting messages with multiple signatures via command line: @@ -3658,7 +3635,7 @@ FEATURES * [#1954](https://github.com/cosmos/cosmos-sdk/issues/1954) New `broadcast` command to broadcast transactions generated offline and signed with the `sign` command. * [cli] [#2220](https://github.com/cosmos/cosmos-sdk/issues/2220) Add `gaiacli config` feature to interactively create CLI config files to reduce the number of required flags * [stake][cli] [#1672](https://github.com/cosmos/cosmos-sdk/issues/1672) Introduced - new commission flags for validator commands `create-validator` and `edit-validator`. + new commission flags for validator commands `create-validator` and `edit-validator`. * [stake][cli] [#1890](https://github.com/cosmos/cosmos-sdk/issues/1890) Add `--genesis-format` flag to `gaiacli tx create-validator` to produce transactions in genesis-friendly format. * [cli][#2554](https://github.com/cosmos/cosmos-sdk/issues/2554) Make `gaiacli keys show` multisig ready. @@ -3677,7 +3654,7 @@ FEATURES * [simulation] [#2349](https://github.com/cosmos/cosmos-sdk/issues/2349) Add time-based future scheduled operations to simulator * [x/auth] [#2376](https://github.com/cosmos/cosmos-sdk/issues/2376) Remove FeePayer() from StdTx * [x/stake] [#1672](https://github.com/cosmos/cosmos-sdk/issues/1672) Implement - basis for the validator commission model. + basis for the validator commission model. * [x/auth] Support account removal in the account mapper. IMPROVEMENTS @@ -3697,7 +3674,7 @@ IMPROVEMENTS * [cli] [#2060](https://github.com/cosmos/cosmos-sdk/issues/2060) removed `--select` from `block` command * [cli] [#2128](https://github.com/cosmos/cosmos-sdk/issues/2128) fixed segfault when exporting directly after `gaiad init` * [cli] [#1255](https://github.com/cosmos/cosmos-sdk/issues/1255) open KeyBase in read-only mode - for query-purpose CLI commands + for query-purpose CLI commands * [docs] Added commands for querying governance deposits, votes and tally * Gaia @@ -3727,8 +3704,8 @@ IMPROVEMENTS * [gaiad] [#1992](https://github.com/cosmos/cosmos-sdk/issues/1992) Add optional flag to `gaiad testnet` to make config directory of daemon (default `gaiad`) and cli (default `gaiacli`) configurable * [x/stake] Add stake `Queriers` for Gaia-lite endpoints. This increases the staking endpoints performance by reusing the staking `keeper` logic for queries. [#2249](https://github.com/cosmos/cosmos-sdk/pull/2149) * [store] [#2017](https://github.com/cosmos/cosmos-sdk/issues/2017) Refactor - gas iterator gas consumption to only consume gas for iterator creation and `Next` - calls which includes dynamic consumption of value length. + gas iterator gas consumption to only consume gas for iterator creation and `Next` + calls which includes dynamic consumption of value length. * [types/decimal] [#2378](https://github.com/cosmos/cosmos-sdk/issues/2378) - Added truncate functionality to decimal * [client] [#1184](https://github.com/cosmos/cosmos-sdk/issues/1184) Remove unused `client/tx/sign.go`. * [tools] [#2464](https://github.com/cosmos/cosmos-sdk/issues/2464) Lock binary dependencies to a specific version @@ -3745,14 +3722,14 @@ BUG FIXES * Gaia * [x/stake] Return correct Tendermint validator update set on `EndBlocker` by not - including non previously bonded validators that have zero power. [#2189](https://github.com/cosmos/cosmos-sdk/issues/2189) + including non previously bonded validators that have zero power. [#2189](https://github.com/cosmos/cosmos-sdk/issues/2189) * [docs] Fixed light client section links * SDK * [#1988](https://github.com/cosmos/cosmos-sdk/issues/1988) Make us compile on OpenBSD (disable ledger) * [#2105](https://github.com/cosmos/cosmos-sdk/issues/2105) Fix DB Iterator leak, which may leak a go routine. * [ledger] [#2064](https://github.com/cosmos/cosmos-sdk/issues/2064) Fix inability to sign and send transactions via the LCD by - loading a Ledger device at runtime. + loading a Ledger device at runtime. * [#2158](https://github.com/cosmos/cosmos-sdk/issues/2158) Fix non-deterministic ordering of validator iteration when slashing in `gov EndBlocker` * [simulation] [#1924](https://github.com/cosmos/cosmos-sdk/issues/1924) Make simulation stop on SIGTERM * [#2388](https://github.com/cosmos/cosmos-sdk/issues/2388) Remove dependency on deprecated tendermint/tmlibs repository. @@ -3762,7 +3739,7 @@ BUG FIXES ## 0.24.2 -*August 22nd, 2018*. +_August 22nd, 2018_. BUG FIXES @@ -3771,7 +3748,7 @@ BUG FIXES ## 0.24.1 -*August 21st, 2018*. +_August 21st, 2018_. BUG FIXES @@ -3780,7 +3757,7 @@ BUG FIXES ## 0.24.0 -*August 13th, 2018*. +_August 13th, 2018_. BREAKING CHANGES @@ -3790,7 +3767,7 @@ BREAKING CHANGES * [x/slashing] [#1866](https://github.com/cosmos/cosmos-sdk/issues/1866) `/slashing/signing_info` takes cosmosvalpub instead of cosmosvaladdr * use time.Time instead of int64 for time. See Tendermint v0.23.0 * Signatures are no longer Amino encoded with prefixes (just encoded as raw - bytes) - see Tendermint v0.23.0 + bytes) - see Tendermint v0.23.0 * Gaia CLI (`gaiacli`) @@ -3798,7 +3775,7 @@ BREAKING CHANGES * [x/stake] [#1828](https://github.com/cosmos/cosmos-sdk/issues/1828) Force user to specify amount on create-validator command by removing default * [x/gov] Change `--proposalID` to `--proposal-id` * [x/stake, x/gov] [#1606](https://github.com/cosmos/cosmos-sdk/issues/1606) Use `--from` instead of adhoc flags like `--address-validator` - and `--proposer` to indicate the sender address. + and `--proposer` to indicate the sender address. * [#1551](https://github.com/cosmos/cosmos-sdk/issues/1551) Remove `--name` completely * Genesis/key creation (`gaiad init`) now supports user-provided key passwords @@ -3806,7 +3783,7 @@ BREAKING CHANGES * [x/stake] Inflation doesn't use rationals in calculation (performance boost) * [x/stake] Persist a map from `addr->pubkey` in the state since BeginBlock - doesn't provide pubkeys. + doesn't provide pubkeys. * [x/gov] [#1781](https://github.com/cosmos/cosmos-sdk/issues/1781) Added tags sub-package, changed tags to use dash-case * [x/gov] [#1688](https://github.com/cosmos/cosmos-sdk/issues/1688) Governance parameters are now stored in globalparams store * [x/gov] [#1859](https://github.com/cosmos/cosmos-sdk/issues/1859) Slash validators who do not vote on a proposal @@ -3820,14 +3797,14 @@ BREAKING CHANGES * [x/auth] Default TxDecoder can be found in `x/auth` rather than baseapp * [client] [#1551](https://github.com/cosmos/cosmos-sdk/issues/1551): Refactored `CoreContext` to `TxContext` and `QueryContext` * Removed all tx related fields and logic (building & signing) to separate - structure `TxContext` in `x/auth/client/context` + structure `TxContext` in `x/auth/client/context` * Tendermint * v0.22.5 -> See [Tendermint PR](https://github.com/tendermint/tendermint/pull/1966) * change all the cryptography imports. * v0.23.0 -> See - [Changelog](https://github.com/tendermint/tendermint/blob/v0.23.0/CHANGELOG.md#0230) - and [SDK PR](https://github.com/cosmos/cosmos-sdk/pull/1927) + [Changelog](https://github.com/tendermint/tendermint/blob/v0.23.0/CHANGELOG.md#0230) + and [SDK PR](https://github.com/cosmos/cosmos-sdk/pull/1927) * BeginBlock no longer includes crypto.Pubkey * use time.Time instead of int64 for time. @@ -3858,7 +3835,7 @@ FEATURES * [baseapp] Initialize validator set on ResponseInitChain * [baseapp] added BaseApp.Seal - ability to seal baseapp parameters once they've been set * [cosmos-sdk-cli] New `cosmos-sdk-cli` tool to quickly initialize a new - SDK-based project + SDK-based project * [scripts] added log output monitoring to DataDog using Ansible scripts IMPROVEMENTS @@ -3905,17 +3882,17 @@ BUG FIXES ## 0.23.1 -*July 27th, 2018*. +_July 27th, 2018_. BUG FIXES * [tendermint] Update to v0.22.8 * [consensus, blockchain] Register the Evidence interface so it can be - marshalled/unmarshalled by the blockchain and consensus reactors + marshalled/unmarshalled by the blockchain and consensus reactors ## 0.23.0 -*July 25th, 2018*. +_July 25th, 2018_. BREAKING CHANGES @@ -3938,7 +3915,7 @@ BUG FIXES ## 0.22.0 -*July 16th, 2018*. +_July 16th, 2018_. BREAKING CHANGES @@ -3957,7 +3934,7 @@ BUG FIXES ## 0.21.1 -*July 14th, 2018*. +_July 14th, 2018_. BUG FIXES @@ -3966,7 +3943,7 @@ BUG FIXES ## 0.21.0 -*July 13th, 2018*. +_July 13th, 2018_. BREAKING CHANGES @@ -3997,7 +3974,7 @@ BUG FIXES ## 0.20.0 -*July 10th, 2018*. +_July 10th, 2018_. BREAKING CHANGES @@ -4009,7 +3986,7 @@ BREAKING CHANGES * Default ports changed from 466xx to 266xx * Amino JSON uses type names instead of prefix bytes * ED25519 addresses are the first 20-bytes of the SHA256 of the raw 32-byte - pubkey (Instead of RIPEMD160) + pubkey (Instead of RIPEMD160) * go-crypto, abci, tmlibs have been merged into Tendermint * The keys sub-module is now in the SDK * Various other fixes @@ -4026,9 +4003,9 @@ BREAKING CHANGES * [cli] Rearranged commands under subcommands * [x/slashing] Update slashing for unbonding period * Slash according to power at time of infraction instead of power at - time of discovery + time of discovery * Iterate through unbonding delegations & redelegations which contributed - to an infraction, slash them proportional to their stake at the time + to an infraction, slash them proportional to their stake at the time * Add REST endpoint to unrevoke a validator previously revoked for downtime * Add REST endpoint to retrieve liveness signing information for a validator * [x/stake] Remove Tick and add EndBlocker @@ -4140,7 +4117,7 @@ BUG FIXES ## 0.19.0 -*June 13, 2018*. +_June 13, 2018_. BREAKING CHANGES @@ -4180,7 +4157,7 @@ FEATURES ## 0.18.0 -*June 9, 2018*. +_June 9, 2018_. BREAKING CHANGES @@ -4190,7 +4167,7 @@ BREAKING CHANGES * [stake] introduce `gaiacli query delegations` * [stake] staking refactor * ValidatorsBonded store now take sorted pubKey-address instead of validator owner-address, - is sorted like Tendermint by pk's address + is sorted like Tendermint by pk's address * store names more understandable * removed temporary ToKick store, just needs a local map! * removed distinction between candidates and validators @@ -4198,7 +4175,7 @@ BREAKING CHANGES * only validators with a status == bonded are actively validating/receiving rewards * Introduction of Unbonding fields, lowlevel logic throughout (not fully implemented with queue) * Introduction of PoolShares type within validators, - replaces three rational fields (BondedShares, UnbondingShares, UnbondedShares + replaces three rational fields (BondedShares, UnbondingShares, UnbondedShares * [x/auth] move stuff specific to auth anteHandler to the auth module rather than the types folder. This includes: * StdTx (and its related stuff i.e. StdSignDoc, etc) * StdFee @@ -4245,26 +4222,26 @@ BUG FIXES ## 0.17.5 -*June 5, 2018*. +_June 5, 2018_. Update to Tendermint v0.19.9 (Fix evidence reactor, mempool deadlock, WAL panic, memory leak) ## 0.17.4 -*May 31, 2018*. +_May 31, 2018_. Update to Tendermint v0.19.7 (WAL fixes and more) ## 0.17.3 -*May 29, 2018*. +_May 29, 2018_. Update to Tendermint v0.19.6 (fix fast-sync halt) ## 0.17.2 -*May 20, 2018*. +_May 20, 2018_. Update to Tendermint v0.19.5 (reduce WAL use, bound the mempool and some rpcs, improve logging) @@ -4570,16 +4547,16 @@ IMPROVEMENTS: * basecoin * `basecoin start` supports all flags that `tendermint node` does, such as - `--rpc.laddr`, `--p2p.seeds`, and `--p2p.skip_upnp` + `--rpc.laddr`, `--p2p.seeds`, and `--p2p.skip_upnp` * fully supports `--log_level` and `--trace` for logger configuration * merkleeyes no longers spams the logs... unless you want it * Example: `basecoin start --log_level="merkleeyes:info,state:info,*:error"` * Example: `basecoin start --log_level="merkleeyes:debug,state:info,*:error"` * basecli * `basecli init` is more intelligent and only complains if there really was - a connected chain, not just random files + a connected chain, not just random files * support `localhost:46657` or `http://localhost:46657` format for nodes, - not just `tcp://localhost:46657` + not just `tcp://localhost:46657` * Add `--genesis` to init to specify chain-id and validator hash * Example: `basecli init --node=localhost:46657 --genesis=$HOME/.basecoin/genesis.json` * `basecli rpc` has a number of methods to easily accept tendermint rpc, and verifies what it can @@ -4588,9 +4565,9 @@ BUG FIXES: * basecli * `basecli query account` accepts hex account address with or without `0x` - prefix + prefix * gives error message when running commands on an unitialized chain, rather - than some unintelligable panic + than some unintelligable panic ## 0.6.0 (June 22, 2017) @@ -4813,4 +4790,4 @@ BUG FIXES: [v0.37.3]: https://github.com/cosmos/cosmos-sdk/releases/tag/v0.37.3 [v0.37.1]: https://github.com/cosmos/cosmos-sdk/releases/tag/v0.37.1 [v0.37.0]: https://github.com/cosmos/cosmos-sdk/releases/tag/v0.37.0 -[v0.36.0]: https://github.com/cosmos/cosmos-sdk/releases/tag/v0.36.0 \ No newline at end of file +[v0.36.0]: https://github.com/cosmos/cosmos-sdk/releases/tag/v0.36.0 From ac27a204a744d472ba4088660e9584374266e5d7 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Thu, 10 Nov 2022 15:31:36 -0600 Subject: [PATCH 16/20] fix changelog, fix adr changelog --- CHANGELOG.md | 2 +- docs/architecture/adr-038-state-listening.md | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c7aa4bcc03..2b10b6081e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,7 +54,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (x/auth) [#13210](https://github.com/cosmos/cosmos-sdk/pull/13210) Add `Query/AccountInfo` endpoint for simplified access to basic account info. * (x/consensus) [#12905](https://github.com/cosmos/cosmos-sdk/pull/12905) Create a new `x/consensus` module that is now responsible for maintaining Tendermint consensus parameters instead of `x/param`. Legacy types remain in order to facilitate parameter migration from the deprecated `x/params`. App developers should ensure that they execute `baseapp.MigrateParams` during their chain upgrade. These legacy types will be removed in a future release. * (client/tx) [#13670](https://github.com/cosmos/cosmos-sdk/pull/13670) Add validation in `BuildUnsignedTx` to prevent simple inclusion of valid mnemonics -* (x/auth) [#13612](https://github.com/cosmos/cosmos-sdk/pull/13612) Add `Query/ModuleAccountByName` endpoint for accessing the module account info by module name. +* [#13473](https://github.com/cosmos/cosmos-sdk/pull/13473) ADR-038: Go plugin system proposal ### Improvements diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index 9cad757ed48..5053375bd56 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -4,9 +4,6 @@ * 11/23/2020: Initial draft * 10/06/2022: Introduce plugin system based on hashicorp/go-plugin -* 10/30/2022: Updated ADR based on review feedback, move listening to `CommitMultiStore` and add a `MemoryListener` [#13516](https://github.com/cosmos/cosmos-sdk/pull/13516) -* 11/01/2022: Add gRPC client, server and plugin implementation examples -* 11/02/2022: Update language on `RegisterStreamingPlugin` function ## Status From 231dacedc75a941dc1d08ec8de1e73f9927ebddb Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj <87285445+egaxhaj@users.noreply.github.com> Date: Fri, 11 Nov 2022 13:56:48 -0600 Subject: [PATCH 17/20] fix formatting on old vs new change Co-authored-by: Aleksandr Bezobchuk --- docs/architecture/adr-038-state-listening.md | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index 5053375bd56..053847cb1b3 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -188,25 +188,7 @@ func (rs *Store) CacheMultiStore() types.CacheMultiStore { ```go func (rs *Store) CacheMultiStoreWithVersion(version int64) (types.CacheMultiStore, error) { - cachedStores := make(map[types.StoreKey]types.CacheWrapper) - for key, store := range rs.stores { - var cacheStore types.KVStore - switch store.GetStoreType() { - case types.StoreTypeIAVL: - // If the store is wrapped with an inter-block cache, we must first unwrap - // it to get the underlying IAVL store. - store = rs.GetCommitKVStore(key) - - // Attempt to lazy-load an already saved IAVL store version. If the - // version does not exist or is pruned, an error should be returned. - var err error - cacheStore, err = store.(*iavl.Store).GetImmutable(version) - if err != nil { - return nil, err - } - default: - cacheStore = store - } + // ... // Wire the listenkv.Store to allow listeners to observe the writes from the cache store, // set same listeners on cache store will observe duplicated writes. From 6d227aeb09e007cf78650375c1721413a6c847b9 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Fri, 11 Nov 2022 14:04:24 -0600 Subject: [PATCH 18/20] pass args directly to inline func --- docs/architecture/adr-038-state-listening.md | 30 +++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index 053847cb1b3..26b25170af3 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -21,7 +21,7 @@ In addition to these request/response queries, it would be beneficial to have a ## Decision -We will modify the `CommitMultiStore` interface and its concrete (`rootmulti`) implementations and introduce a new `listenkv.Store` to allow listening to state changes in underlying KVStores. +We will modify the store layer to allow listening to state changes in underlying KVStores. We will introduce a plugin system for configuring and running streaming services that write these state changes and their surrounding ABCI message context to different destinations. ### Listening @@ -246,12 +246,11 @@ func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeg os.Exit(1) } } else { - rek, rez := req, res - go func() { - if err := app.abciListener.ListenBeginBlock(ctx, rek, rez); err != nil { + go func(req abci.RequestBeginBlock, res abci.ResponseBeginBlock) { + if err := app.abciListener.ListenBeginBlock(ctx, req, res); err != nil { app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", err) } - }() + }(req, res) } } @@ -274,14 +273,13 @@ func (app *BaseApp) EndBlock(req abci.RequestEndBlock) (res abci.ResponseEndBloc os.Exit(1) } } else { - rek, rez := req, res - go func() { + go func(req abci.RequestEndBlock, res abci.ResponseEndBlock) { if reqErr == nil && resErr == nil { - if err := app.abciListener.ListenEndBlock(blockHeight, rek, rez); err != nil { + if err := app.abciListener.ListenEndBlock(blockHeight, req, res); err != nil { app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", err) } } - }() + }(req, res) } } @@ -304,14 +302,13 @@ func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx os.Exit(1) } } else { - rek, rez := req, abciRes - go func() { + go func(req abci.RequestDeliverTx, res abci.ResponseDeliverTx) { if reqErr == nil && resErr == nil { - if err := app.abciListener.ListenDeliverTx(blockHeight, rek, rez); err != nil { + if err := app.abciListener.ListenDeliverTx(blockHeight, req, res); err != nil { app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", err) } } - }() + }(req, abciRes) } } }() @@ -343,12 +340,11 @@ func (app *BaseApp) Commit() abci.ResponseCommit { os.Exit(1) } } else { - rez := res - go func() { - if err := app.abciListener.ListenCommit(ctx, rez, changeSet); err != nil { + go func(res abci.ResponseCommit, changeSet []store.StoreKVPair) { + if err := app.abciListener.ListenCommit(ctx, res, changeSet); err != nil { app.logger.Error("ListenCommit listening hook failed", "height", blockHeight, "err", err) } - }() + }(res, changeSet) } } From 3312ee94dc71c040b55cfa4f26eca819b0a60f90 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Fri, 18 Nov 2022 14:48:28 -0600 Subject: [PATCH 19/20] deterministic change set pop, multi plugin loading, use base context.Context --- docs/architecture/adr-038-state-listening.md | 70 +++++++++++--------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index 26b25170af3..84c79bef486 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -163,6 +163,9 @@ func (rs *Store) PopStateCache() []types.StoreKVPair { for _, ls := range rs.listeners { cache = append(cache, ls.PopStateCache()...) } + sort.SliceStable(cache, func(i, j int) bool { + return cache[i].StoreKey < cache[j].StoreKey + }) return cache } ``` @@ -216,13 +219,13 @@ so that the service can group the state changes with the ABCI requests. // ABCIListener is the interface that we're exposing as a streaming service. type ABCIListener interface { // ListenBeginBlock updates the streaming service with the latest BeginBlock messages - ListenBeginBlock(ctx types.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error + ListenBeginBlock(ctx context.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error // ListenEndBlock updates the steaming service with the latest EndBlock messages - ListenEndBlock(ctx types.Context, req abci.RequestEndBlock, res abci.ResponseEndBlock) error + ListenEndBlock(ctx context.Context, req abci.RequestEndBlock, res abci.ResponseEndBlock) error // ListenDeliverTx updates the steaming service with the latest DeliverTx messages - ListenDeliverTx(ctx types.Context, req abci.RequestDeliverTx, res abci.ResponseDeliverTx) error + ListenDeliverTx(ctx context.Context, req abci.RequestDeliverTx, res abci.ResponseDeliverTx) error // ListenCommit updates the steaming service with the latest Commit messages and state changes - ListenCommit(ctx types.Context, res abci.ResponseCommit, changeSet []store.StoreKVPair) error + ListenCommit(ctx context.Context, res abci.ResponseCommit, changeSet []store.StoreKVPair) error } ``` @@ -581,12 +584,13 @@ is helper function and not a requirement. Plugin registration can easily be move ```go // baseapp/streaming.go -// RegisterStreamingPlugin registers the ABCI streaming service provided by the streaming plugin. +// RegisterStreamingPlugin registers streaming plugins with the App. +// This method returns an error if a plugin is not supported. func RegisterStreamingPlugin( bApp *BaseApp, appOpts servertypes.AppOptions, keys map[string]*types.KVStoreKey, - streamingService interface{}, + streamingPlugin interface{}, ) error { switch t := streamingPlugin.(type) { case ABCIListener: @@ -609,7 +613,7 @@ func registerABCIListenerPlugin( stopNodeOnErr := cast.ToBool(appOpts.Get(stopNodeOnErrKey)) keysKey := fmt.Sprintf("%s.%s", StreamingTomlKey, StreamingKeysTomlKey) exposeKeysStr := cast.ToStringSlice(appOpts.Get(keysKey)) - bApp.cms.AddListeners(exposeStoreKeysSorted(exposeKeysStr, keys)) + bApp.cms.AddListeners(exposeStoreKeys(exposeKeysStr, keys)) bApp.abciListener = abciListener bApp.stopNodeOnStreamingErr = stopNodeOnErr } @@ -625,7 +629,7 @@ func exposeAll(list []string) bool { return false } -func exposeStoreKeysSorted(keysStr []string, keys map[string]*types.KVStoreKey) []types.StoreKey { +func exposeStoreKeys(keysStr []string, keys map[string]*types.KVStoreKey) []types.StoreKey { var exposeStoreKeys []types.StoreKey if exposeAll(keysStr) { exposeStoreKeys = make([]types.StoreKey, 0, len(keys)) @@ -640,10 +644,6 @@ func exposeStoreKeysSorted(keysStr []string, keys map[string]*types.KVStoreKey) } } } - // sort storeKeys for deterministic output - sort.SliceStable(exposeStoreKeys, func(i, j int) bool { - return strings.Compare(exposeStoreKeys[i].Name(), exposeStoreKeys[j].Name()) < 0 - }) return exposeStoreKeys } @@ -674,19 +674,21 @@ func NewSimApp( ... - // enable streaming - enableKey := fmt.Sprintf("%s.%s", baseapp.StreamingTomlKey, baseapp.StreamingEnableTomlKey) - if enable := cast.ToBool(appOpts.Get(enableKey)); enable { - pluginKey := fmt.Sprintf("%s.%s", baseapp.StreamingTomlKey, baseapp.StreamingPluginTomlKey) - pluginName := cast.ToString(appOpts.Get(pluginKey)) - logLevel := cast.ToString(appOpts.Get(flags.FlagLogLevel)) - plugin, err := streaming.NewStreamingPlugin(pluginName, logLevel) - if err != nil { - tmos.Exit(err.Error()) - } - if err := baseapp.RegisterStreamingPlugin(bApp, appOpts, keys, plugin); err != nil { - tmos.Exit(err.Error()) - } + // register streaming services + streamingCfg := cast.ToStringMap(appOpts.Get(baseapp.StreamingTomlKey)) + for service := range streamingCfg { + pluginKey := fmt.Sprintf("%s.%s.%s", baseapp.StreamingTomlKey, service, baseapp.StreamingPluginTomlKey) + pluginName := strings.TrimSpace(cast.ToString(appOpts.Get(pluginKey))) + if len(pluginName) > 0 { + logLevel := cast.ToString(appOpts.Get(flags.FlagLogLevel)) + plugin, err := streaming.NewStreamingPlugin(pluginName, logLevel) + if err != nil { + tmos.Exit(err.Error()) + } + if err := baseapp.RegisterStreamingPlugin(bApp, appOpts, keys, plugin); err != nil { + tmos.Exit(err.Error()) + } + } } return app @@ -700,11 +702,11 @@ The plugin system will be configured within an App's TOML configuration files. # gRPC streaming [streaming] -# StreamingEnale defines if the gRPC streaming plugins should be enabled. -enable = true +# ABCI streaming service +[streaming.abci] # The plugin version to use for ABCI listening -plugin = "grpc_abci_v1" +plugin = "abci_v1" # List of kv store keys to listen to for state changes. # Set to ["*"] to expose all keys. @@ -714,9 +716,15 @@ keys = ["*"] stop-node-on-err = true ``` -There will be four parameters for configuring the streaming plugin system: `streaming.enable`, `streaming.keys`, `streaming.plugin` and `streaming.stop-node-on-err`. -`streaming.enable` is a bool that turns on or off the streaming service, `streaming.keys` is a set of store keys for stores it listens to, -`streaming.plugin` is the name of the plugin we want to use for streaming and `streaming.stop-node-on-err` is a bool that stops the node when true and operates in a fire-and-forget mode when false. +There will be three parameters for configuring `ABCIListener` plugin: `streaming.abci.plugin`, `streaming.abci.keys` and `streaming.abci.stop-node-on-err`. +`streaming.abci.plugin` is the name of the plugin we want to use for streaming, `streaming.abci.keys` is a set of store keys for stores it listens to, + and `streaming.abci.stop-node-on-err` is a bool that stops the node when true and operates in a fire-and-forget mode when false. + +The configuration above support additional streaming plugins by adding the plugin to the `[streaming]` configuration section +and registering the plugin with `RegisterStreamingPlugin` helper function. + +Note the that each plugin must include `streaming.{service}.plugin` property as it is a requirement for doing the lookup and registration of the plugin +with the App. All other properties are unique to the individual services. #### Encoding and decoding streams From 4f6e8c069d761881bf2cdbc61b1cf98f4f861cf7 Mon Sep 17 00:00:00 2001 From: Ergels Gaxhaj Date: Mon, 21 Nov 2022 15:37:37 -0600 Subject: [PATCH 20/20] support in-process and out-of-process listeners --- docs/architecture/adr-038-state-listening.md | 136 +++++++++++++------ 1 file changed, 92 insertions(+), 44 deletions(-) diff --git a/docs/architecture/adr-038-state-listening.md b/docs/architecture/adr-038-state-listening.md index 84c79bef486..af8f2bca987 100644 --- a/docs/architecture/adr-038-state-listening.md +++ b/docs/architecture/adr-038-state-listening.md @@ -229,6 +229,36 @@ type ABCIListener interface { } ``` +#### BaseApp Registration + +We will add a new method to the `BaseApp` to enable the registration of `StreamingService`s: + + ```go + // SetStreamingService is used to set a streaming service into the BaseApp hooks and load the listeners into the multistore +func (app *BaseApp) SetStreamingService(s ABCIListener) { + // register the StreamingService within the BaseApp + // BaseApp will pass BeginBlock, DeliverTx, and EndBlock requests and responses to the streaming services to update their ABCI context + app.abciListeners = append(app.abciListeners, s) +} +``` + +We will add two new fields to the `BaseApp` struct: +```go +type BaseApp struct { + + ... + + // abciListenersAsync for determining if abciListeners will run asynchronously. + // When abciListenersAsync=false and stopNodeOnABCIListenerErr=false listeners will run synchronized but will not stop the node. + // When abciListenersAsync=true stopNodeOnABCIListenerErr will be ignored. + abciListenersAsync bool + + // stopNodeOnABCIListenerErr halts the node when ABCI streaming service listening results in an error. + // stopNodeOnABCIListenerErr=true must be paired with abciListenersAsync=false. + stopNodeOnABCIListenerErr bool +} +``` + #### ABCI Event Hooks We will modify the `BeginBlock`, `EndBlock`, `DeliverTx` and `Commit` methods to pass ABCI requests and responses @@ -240,20 +270,22 @@ func (app *BaseApp) BeginBlock(req abci.RequestBeginBlock) (res abci.ResponseBeg ... // call the streaming service hook with the BeginBlock messages - if app.abciListener != nil { + for _, abciListener := range app.abciListeners { ctx := app.deliverState.ctx blockHeight := ctx.BlockHeight() - if app.stopNodeOnStreamingErr { - if err := app.abciListener.ListenBeginBlock(ctx, req, res); err != nil { - app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", err) - os.Exit(1) - } - } else { + if app.abciListenersAsync { go func(req abci.RequestBeginBlock, res abci.ResponseBeginBlock) { if err := app.abciListener.ListenBeginBlock(ctx, req, res); err != nil { - app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", err) + app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", err) } }(req, res) + } else { + if err := app.abciListener.ListenBeginBlock(ctx, req, res); err != nil { + app.logger.Error("BeginBlock listening hook failed", "height", blockHeight, "err", err) + if app.stopNodeOnABCIListenerErr { + os.Exit(1) + } + } } } @@ -267,22 +299,22 @@ func (app *BaseApp) EndBlock(req abci.RequestEndBlock) (res abci.ResponseEndBloc ... // call the streaming service hook with the EndBlock messages - if app.abciListener != nil { + for _, abciListener := range app.abciListeners { ctx := app.deliverState.ctx blockHeight := ctx.BlockHeight() - if app.stopNodeOnStreamingErr { - if err := app.abciListener.ListenEndBlock(blockHeight, req, res); err != nil { - app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", err) - os.Exit(1) - } - } else { + if app.abciListenersAsync { go func(req abci.RequestEndBlock, res abci.ResponseEndBlock) { - if reqErr == nil && resErr == nil { - if err := app.abciListener.ListenEndBlock(blockHeight, req, res); err != nil { - app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", err) - } + if err := app.abciListener.ListenEndBlock(blockHeight, req, res); err != nil { + app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", err) } }(req, res) + } else { + if err := app.abciListener.ListenEndBlock(blockHeight, req, res); err != nil { + app.logger.Error("EndBlock listening hook failed", "height", blockHeight, "err", err) + if app.stopNodeOnABCIListenerErr { + os.Exit(1) + } + } } } @@ -296,22 +328,22 @@ func (app *BaseApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx var abciRes abci.ResponseDeliverTx defer func() { // call the streaming service hook with the EndBlock messages - if app.abciListener != nil { + for _, abciListener := range app.abciListeners { ctx := app.deliverState.ctx blockHeight := ctx.BlockHeight() - if app.stopNodeOnStreamingErr { - if err := app.abciListener.ListenDeliverTx(blockHeight, req, res); err != nil { - app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", err) - os.Exit(1) - } - } else { + if app.abciListenersAsync { go func(req abci.RequestDeliverTx, res abci.ResponseDeliverTx) { - if reqErr == nil && resErr == nil { - if err := app.abciListener.ListenDeliverTx(blockHeight, req, res); err != nil { - app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", err) - } + if err := app.abciListener.ListenDeliverTx(blockHeight, req, res); err != nil { + app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", err) } }(req, abciRes) + } else { + if err := app.abciListener.ListenDeliverTx(blockHeight, req, res); err != nil { + app.logger.Error("DeliverTx listening hook failed", "height", blockHeight, "err", err) + if app.stopNodeOnABCIListenerErr { + os.Exit(1) + } + } } } }() @@ -333,21 +365,23 @@ func (app *BaseApp) Commit() abci.ResponseCommit { } // call the streaming service hook with the Commit messages - if app.abciListener != nil { + for _, abciListener := range app.abciListeners { ctx := app.deliverState.ctx blockHeight := ctx.BlockHeight() changeSet := app.cms.PopStateCache() - if app.stopNodeOnStreamingErr { - if err := app.abciListener.ListenCommit(ctx, res, changeSet); err != nil { - app.logger.Error("ListenCommit listening hook failed", "height", blockHeight, "err", err) - os.Exit(1) - } - } else { + if app.abciListenersAsync { go func(res abci.ResponseCommit, changeSet []store.StoreKVPair) { if err := app.abciListener.ListenCommit(ctx, res, changeSet); err != nil { app.logger.Error("ListenCommit listening hook failed", "height", blockHeight, "err", err) } }(res, changeSet) + } else { + if err := app.abciListener.ListenCommit(ctx, res, changeSet); err != nil { + app.logger.Error("ListenCommit listening hook failed", "height", blockHeight, "err", err) + if app.stopNodeOnABCIListenerErr { + os.Exit(1) + } + } } } @@ -609,13 +643,17 @@ func registerABCIListenerPlugin( keys map[string]*store.KVStoreKey, abciListener ABCIListener, ) { - stopNodeOnErrKey := fmt.Sprintf("%s.%s", StreamingTomlKey, StreamingStopNodeOnErrTomlKey) + asyncKey := fmt.Sprintf("%s.%s.%s", StreamingTomlKey, StreamingABCITomlKey, StreamingABCIAsync) + async := cast.ToBool(appOpts.Get(asyncKey)) + stopNodeOnErrKey := fmt.Sprintf("%s.%s.%s", StreamingTomlKey, StreamingABCITomlKey, StreamingABCIStopNodeOnErrTomlKey) stopNodeOnErr := cast.ToBool(appOpts.Get(stopNodeOnErrKey)) - keysKey := fmt.Sprintf("%s.%s", StreamingTomlKey, StreamingKeysTomlKey) + keysKey := fmt.Sprintf("%s.%s.%s", StreamingTomlKey, StreamingABCITomlKey, StreamingABCIKeysTomlKey) exposeKeysStr := cast.ToStringSlice(appOpts.Get(keysKey)) - bApp.cms.AddListeners(exposeStoreKeys(exposeKeysStr, keys)) - bApp.abciListener = abciListener - bApp.stopNodeOnStreamingErr = stopNodeOnErr + exposedKeys := exposeStoreKeysSorted(exposeKeysStr, keys) + bApp.cms.AddListeners(exposedKeys) + bApp.SetStreamingService(abciListener) + bApp.stopNodeOnABCIListenerErr = stopNodeOnErr + bApp.abciListenersAsync = async } ``` @@ -644,6 +682,10 @@ func exposeStoreKeys(keysStr []string, keys map[string]*types.KVStoreKey) []type } } } + // sort storeKeys for deterministic output + sort.SliceStable(exposeStoreKeys, func(i, j int) bool { + return exposeStoreKeys[i].Name() < exposeStoreKeys[j].Name() + }) return exposeStoreKeys } @@ -712,13 +754,19 @@ plugin = "abci_v1" # Set to ["*"] to expose all keys. keys = ["*"] +# Enable abciListeners to run asynchronously. +# When abciListenersAsync=false and stopNodeOnABCIListenerErr=false listeners will run synchronized but will not stop the node. +# When abciListenersAsync=true stopNodeOnABCIListenerErr will be ignored. +async = false + # Whether to stop the node on message deliver error. stop-node-on-err = true ``` -There will be three parameters for configuring `ABCIListener` plugin: `streaming.abci.plugin`, `streaming.abci.keys` and `streaming.abci.stop-node-on-err`. +There will be four parameters for configuring `ABCIListener` plugin: `streaming.abci.plugin`, `streaming.abci.keys`, `streaming.abci.async` and `streaming.abci.stop-node-on-err`. `streaming.abci.plugin` is the name of the plugin we want to use for streaming, `streaming.abci.keys` is a set of store keys for stores it listens to, - and `streaming.abci.stop-node-on-err` is a bool that stops the node when true and operates in a fire-and-forget mode when false. +`streaming.abci.async` is bool enabling asynchronous listening and `streaming.abci.stop-node-on-err` is a bool that stops the node when true and when operating +on synchronized mode `streaming.abci.async=false`. Note that `streaming.abci.stop-node-on-err=true` will be ignored if `streaming.abci.async=true`. The configuration above support additional streaming plugins by adding the plugin to the `[streaming]` configuration section and registering the plugin with `RegisterStreamingPlugin` helper function.