Merge PR #5610: proto: migrate x/distr to use hybrid codec

CHANGELOG.md

@@ -83,6 +83,15 @@ for JSON encoding.
   * Every reference of `crypto.Pubkey` in context of a `Validator` is now of type string. `GetPubKeyFromBech32` must be used to get the `crypto.Pubkey`.
   * The `Keeper` constructor now takes a `codec.Marshaler` instead of a concrete Amino codec. This exact type
   provided is specified by `ModuleCdc`.
+* (distr) [\#5610](https://github.com/cosmos/cosmos-sdk/pull/5610) Migrate the `x/distribution` module to use Protocol Buffer for state
+serialization instead of Amino. The exact codec used is `codec.HybridCodec` which utilizes Protobuf for binary encoding and Amino
+for JSON encoding.
+  * `ValidatorHistoricalRewards.ReferenceCount` is now of types `uint32` instead of `uint16`.
+  * `ValidatorSlashEvents` is now a struct with `slashevents`.
+  * `ValidatorOutstandingRewards` is now a struct with `rewards`.
+  * `ValidatorAccumulatedCommission` is now a struct with `commission`.
+  * The `Keeper` constructor now takes a `codec.Marshaler` instead of a concrete Amino codec. This exact type
+  provided is specified by `ModuleCdc`.
 
 ### Improvements
 

Makefile

@@ -238,4 +238,25 @@ proto-lint:
 proto-check-breaking:
 	@buf check breaking --against-input '.git#branch=master'
 
-.PHONY: proto-all proto-gen proto-lint proto-check-breaking
+# Origin
+# TODO: Update to the version of Tendermint that is being used in go.mod
+version_branch = v0.33.0
+tendermint = https://raw.githubusercontent.com/tendermint/tendermint/$(version_branch)
+
+# Outputs
+tmkv = third_party/proto/tendermint/libs/kv/types.proto
+tmmerkle = third_party/proto/tendermint/crypto/merkle/merkle.proto
+tmabci = third_party/proto/tendermint/abci/types/types.proto
+
+# You *only* need to run this to rebuild protobufs from the tendermint source
+proto-update-tendermint:
+	@curl $(tendermint)/abci/types/types.proto > $(tmabci)
+	sed -i '' '8,9 s|github.com/tendermint|third_party/proto|g' $(tmabci)
+	sed -i '' '7 s|github.com/gogo/protobuf|third_party/proto|' $(tmabci)
+	@curl $(tendermint)/libs/kv/types.proto > $(tmkv)
+	sed -i '' 's|github.com/gogo/protobuf|third_party/proto|' $(tmkv)
+	@curl $(tendermint)/crypto/merkle/merkle.proto > $(tmmerkle)
+	sed -i '' '7 s|github.com/gogo/protobuf|third_party/proto|' $(tmmerkle)
+
+
+.PHONY: proto-all proto-gen proto-lint proto-check-breaking proto-update-tendermint

contrib/devtools/Makefile

@@ -48,7 +48,7 @@ UNAME_M ?= $(shell uname -m)
 GOPATH ?= $(shell $(GO) env GOPATH)
 GITHUBDIR := $(GOPATH)$(FS)src$(FS)github.com
 
-BUF_VERSION ?= 0.4.0
+BUF_VERSION ?= 0.7.0
 
 TOOLS_DESTDIR  ?= $(GOPATH)/bin
 STATIK         = $(TOOLS_DESTDIR)/statik

simapp/app.go

@@ -179,7 +179,7 @@ func NewSimApp(
 		app.SupplyKeeper, auth.FeeCollectorName,
 	)
 	app.DistrKeeper = distr.NewKeeper(
-		app.cdc, keys[distr.StoreKey], app.subspaces[distr.ModuleName], app.BankKeeper, &stakingKeeper,
+		appCodec.Distribution, keys[distr.StoreKey], app.subspaces[distr.ModuleName], app.BankKeeper, &stakingKeeper,
 		app.SupplyKeeper, auth.FeeCollectorName, app.ModuleAccountAddrs(),
 	)
 	app.SlashingKeeper = slashing.NewKeeper(

simapp/codec.go

@@ -4,6 +4,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/codec"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/x/auth/vesting"
+	distr "github.com/cosmos/cosmos-sdk/x/distribution"
 	"github.com/cosmos/cosmos-sdk/x/staking"
 )
 
@@ -12,15 +13,17 @@ import (
 type AppCodec struct {
 	amino *codec.Codec
 
-	Staking *staking.Codec
+	Staking      *staking.Codec
+	Distribution *distr.Codec
 }
 
 func NewAppCodec() *AppCodec {
 	amino := MakeCodec()
 
 	return &AppCodec{
-		amino:   amino,
-		Staking: staking.NewCodec(amino),
+		amino:        amino,
+		Staking:      staking.NewCodec(amino),
+		Distribution: distr.NewCodec(amino),
 	}
 }
 

simapp/export.go

@@ -87,9 +87,8 @@ func (app *SimApp) prepForZeroHeightGenesis(ctx sdk.Context, jailWhiteList []str
 
 	// reinitialize all validators
 	app.StakingKeeper.IterateValidators(ctx, func(_ int64, val exported.ValidatorI) (stop bool) {
-
 		// donate any unwithdrawn outstanding reward fraction tokens to the community pool
-		scraps := app.DistrKeeper.GetValidatorOutstandingRewards(ctx, val.GetOperator())
+		scraps := app.DistrKeeper.GetValidatorOutstandingRewardsCoins(ctx, val.GetOperator())
 		feePool := app.DistrKeeper.GetFeePool(ctx)
 		feePool.CommunityPool = feePool.CommunityPool.Add(scraps...)
 		app.DistrKeeper.SetFeePool(ctx, feePool)

third_party/proto/tendermint/abci/types/types.proto

@@ -18,7 +18,6 @@ option (gogoproto.marshaler_all) = true;
 option (gogoproto.unmarshaler_all) = true;
 option (gogoproto.sizer_all) = true;
 option (gogoproto.goproto_registration) = true;
-
 // Generate tests
 option (gogoproto.populate_all) = true;
 option (gogoproto.equal_all) = true;
@@ -43,9 +42,12 @@ message Request {
   }
 }
 
-message RequestEcho { string message = 1; }
+message RequestEcho {
+  string message = 1;
+}
 
-message RequestFlush {}
+message RequestFlush {
+}
 
 message RequestInfo {
   string version = 1;
@@ -60,11 +62,10 @@ message RequestSetOption {
 }
 
 message RequestInitChain {
-  google.protobuf.Timestamp time = 1
-      [ (gogoproto.nullable) = false, (gogoproto.stdtime) = true ];
+  google.protobuf.Timestamp time = 1  [(gogoproto.nullable)=false, (gogoproto.stdtime)=true];
   string chain_id = 2;
   ConsensusParams consensus_params = 3;
-  repeated ValidatorUpdate validators = 4 [ (gogoproto.nullable) = false ];
+  repeated ValidatorUpdate validators = 4  [(gogoproto.nullable)=false];
   bytes app_state_bytes = 5;
 }
 
@@ -77,9 +78,9 @@ message RequestQuery {
 
 message RequestBeginBlock {
   bytes hash = 1;
-  Header header = 2 [ (gogoproto.nullable) = false ];
-  LastCommitInfo last_commit_info = 3 [ (gogoproto.nullable) = false ];
-  repeated Evidence byzantine_validators = 4 [ (gogoproto.nullable) = false ];
+  Header header = 2 [(gogoproto.nullable)=false];
+  LastCommitInfo last_commit_info = 3 [(gogoproto.nullable)=false];
+  repeated Evidence byzantine_validators = 4 [(gogoproto.nullable)=false];
 }
 
 enum CheckTxType {
@@ -92,11 +93,16 @@ message RequestCheckTx {
   CheckTxType type = 2;
 }
 
-message RequestDeliverTx { bytes tx = 1; }
+message RequestDeliverTx {
+  bytes tx = 1;
+}
 
-message RequestEndBlock { int64 height = 1; }
+message RequestEndBlock {
+  int64 height = 1;
+}
 
-message RequestCommit {}
+message RequestCommit {
+}
 
 //----------------------------------------
 // Response types
@@ -119,11 +125,16 @@ message Response {
 }
 
 // nondeterministic
-message ResponseException { string error = 1; }
+message ResponseException {
+  string error = 1;
+}
 
-message ResponseEcho { string message = 1; }
+message ResponseEcho {
+  string message = 1;
+}
 
-message ResponseFlush {}
+message ResponseFlush {
+}
 
 message ResponseInfo {
   string data = 1;
@@ -145,13 +156,13 @@ message ResponseSetOption {
 
 message ResponseInitChain {
   ConsensusParams consensus_params = 1;
-  repeated ValidatorUpdate validators = 2 [ (gogoproto.nullable) = false ];
+  repeated ValidatorUpdate validators = 2 [(gogoproto.nullable)=false];
 }
 
 message ResponseQuery {
   uint32 code = 1;
   // bytes data = 2; // use "value" instead.
-  string log = 3;  // nondeterministic
+  string log = 3; // nondeterministic
   string info = 4; // nondeterministic
   int64 index = 5;
   bytes key = 6;
@@ -162,48 +173,35 @@ message ResponseQuery {
 }
 
 message ResponseBeginBlock {
-  repeated Event events = 1 [
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "events,omitempty"
-  ];
+  repeated Event events = 1 [(gogoproto.nullable)=false, (gogoproto.jsontag)="events,omitempty"];
 }
 
 message ResponseCheckTx {
   uint32 code = 1;
   bytes data = 2;
-  string log = 3;  // nondeterministic
+  string log = 3; // nondeterministic
   string info = 4; // nondeterministic
-  int64 gas_wanted = 5;
+  int64 gas_wanted  = 5;
   int64 gas_used = 6;
-  repeated Event events = 7 [
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "events,omitempty"
-  ];
+  repeated Event events = 7 [(gogoproto.nullable)=false, (gogoproto.jsontag)="events,omitempty"];
   string codespace = 8;
 }
 
 message ResponseDeliverTx {
   uint32 code = 1;
   bytes data = 2;
-  string log = 3;  // nondeterministic
+  string log = 3; // nondeterministic
   string info = 4; // nondeterministic
   int64 gas_wanted = 5;
   int64 gas_used = 6;
-  repeated Event events = 7 [
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "events,omitempty"
-  ];
+  repeated Event events = 7 [(gogoproto.nullable)=false, (gogoproto.jsontag)="events,omitempty"];
   string codespace = 8;
 }
 
 message ResponseEndBlock {
-  repeated ValidatorUpdate validator_updates = 1
-      [ (gogoproto.nullable) = false ];
+  repeated ValidatorUpdate validator_updates = 1 [(gogoproto.nullable)=false];
   ConsensusParams consensus_param_updates = 2;
-  repeated Event events = 3 [
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "events,omitempty"
-  ];
+  repeated Event events = 3 [(gogoproto.nullable)=false, (gogoproto.jsontag)="events,omitempty"];
 }
 
 message ResponseCommit {
@@ -233,51 +231,47 @@ message BlockParams {
 message EvidenceParams {
   // Note: must be greater than 0
   int64 max_age_num_blocks = 1;
-  google.protobuf.Duration max_age_duration = 2
-      [ (gogoproto.nullable) = false, (gogoproto.stdduration) = true ];
+  google.protobuf.Duration max_age_duration = 2 [(gogoproto.nullable)=false, (gogoproto.stdduration)=true];
 }
 
 // ValidatorParams contains limits on validators.
-message ValidatorParams { repeated string pub_key_types = 1; }
+message ValidatorParams {
+  repeated string pub_key_types = 1;
+}
 
 message LastCommitInfo {
   int32 round = 1;
-  repeated VoteInfo votes = 2 [ (gogoproto.nullable) = false ];
+  repeated VoteInfo votes = 2 [(gogoproto.nullable)=false];
 }
 
 message Event {
   string type = 1;
-  repeated tendermint.libs.kv.Pair attributes = 2 [
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "attributes,omitempty"
-  ];
+  repeated tendermint.libs.kv.Pair attributes = 2 [(gogoproto.nullable)=false, (gogoproto.jsontag)="attributes,omitempty"];
 }
 
 //----------------------------------------
 // Blockchain Types
 
 message Header {
   // basic block info
-  Version version = 1 [ (gogoproto.nullable) = false ];
-  string chain_id = 2 [ (gogoproto.customname) = "ChainID" ];
+  Version version = 1 [(gogoproto.nullable)=false];
+  string chain_id = 2 [(gogoproto.customname)="ChainID"];
   int64 height = 3;
-  google.protobuf.Timestamp time = 4
-      [ (gogoproto.nullable) = false, (gogoproto.stdtime) = true ];
+  google.protobuf.Timestamp time = 4 [(gogoproto.nullable)=false, (gogoproto.stdtime)=true];
 
   // prev block info
-  BlockID last_block_id = 5 [ (gogoproto.nullable) = false ];
+  BlockID last_block_id = 5 [(gogoproto.nullable)=false];
 
   // hashes of block data
   bytes last_commit_hash = 6; // commit from validators from the last block
   bytes data_hash = 7;        // transactions
 
   // hashes from the app output from the prev block
-  bytes validators_hash = 8;      // validators for the current block
-  bytes next_validators_hash = 9; // validators for the next block
-  bytes consensus_hash = 10;      // consensus params for current block
-  bytes app_hash = 11;            // state after txs from the previous block
-  bytes last_results_hash =
-      12; // root hash of all results from the txs from the previous block
+  bytes validators_hash = 8;   // validators for the current block
+  bytes next_validators_hash = 9;   // validators for the next block
+  bytes consensus_hash = 10;   // consensus params for current block
+  bytes app_hash = 11;         // state after txs from the previous block
+  bytes last_results_hash = 12;// root hash of all results from the txs from the previous block
 
   // consensus info
   bytes evidence_hash = 13;    // evidence included in the block
@@ -289,9 +283,10 @@ message Version {
   uint64 App = 2;
 }
 
+
 message BlockID {
   bytes hash = 1;
-  PartSetHeader parts_header = 2 [ (gogoproto.nullable) = false ];
+  PartSetHeader parts_header = 2 [(gogoproto.nullable)=false];
 }
 
 message PartSetHeader {
@@ -302,41 +297,40 @@ message PartSetHeader {
 // Validator
 message Validator {
   bytes address = 1;
-  // PubKey pub_key = 2 [(gogoproto.nullable)=false];
+  //PubKey pub_key = 2 [(gogoproto.nullable)=false];
   int64 power = 3;
 }
 
 // ValidatorUpdate
 message ValidatorUpdate {
-  PubKey pub_key = 1 [ (gogoproto.nullable) = false ];
+  PubKey pub_key = 1 [(gogoproto.nullable)=false];
   int64 power = 2;
 }
 
 // VoteInfo
 message VoteInfo {
-  Validator validator = 1 [ (gogoproto.nullable) = false ];
+  Validator validator = 1 [(gogoproto.nullable)=false];
   bool signed_last_block = 2;
 }
 
 message PubKey {
   string type = 1;
-  bytes data = 2;
+  bytes  data = 2;
 }
 
 message Evidence {
   string type = 1;
-  Validator validator = 2 [ (gogoproto.nullable) = false ];
+  Validator validator = 2 [(gogoproto.nullable)=false];
   int64 height = 3;
-  google.protobuf.Timestamp time = 4
-      [ (gogoproto.nullable) = false, (gogoproto.stdtime) = true ];
+  google.protobuf.Timestamp time = 4 [(gogoproto.nullable)=false, (gogoproto.stdtime)=true];
   int64 total_voting_power = 5;
 }
 
 //----------------------------------------
 // Service Definition
 
 service ABCIApplication {
-  rpc Echo(RequestEcho) returns (ResponseEcho);
+  rpc Echo(RequestEcho) returns (ResponseEcho) ;
   rpc Flush(RequestFlush) returns (ResponseFlush);
   rpc Info(RequestInfo) returns (ResponseInfo);
   rpc SetOption(RequestSetOption) returns (ResponseSetOption);