add reverse iteration to pagination (#8875)

* WIP

* add tests

* add tests

* fix lint

* fix pagination

* add proto message doc

* fix filtered_pagination

* fix test

* cleanup

* add reverse flag to pagination

* changelog

* Update client/flags/flags.go

* Update CHANGELOG.md

Co-authored-by: Alessio Treglia <alessio@tendermint.com>
Co-authored-by: Federico Kunze <31522760+fedekunze@users.noreply.github.com>

CHANGELOG.md

@@ -41,6 +41,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * [\#8559](https://github.com/cosmos/cosmos-sdk/pull/8559) Added Protobuf compatible secp256r1 ECDSA signatures.
 * [\#8786](https://github.com/cosmos/cosmos-sdk/pull/8786) Enabled secp256r1 in x/auth.
 * (rosetta) [\#8729](https://github.com/cosmos/cosmos-sdk/pull/8729) Data API fully supports balance tracking. Construction API can now construct any message supported by the application.
+* [\#8754](https://github.com/cosmos/cosmos-sdk/pull/8875) Added support for reverse iteration to pagination.
 
 ### Client Breaking Changes
 

client/flags/flags.go

@@ -71,6 +71,7 @@ const (
 	FlagTimeoutHeight    = "timeout-height"
 	FlagKeyAlgorithm     = "algo"
 	FlagFeeAccount       = "fee-account"
+	FlagReverse          = "reverse"
 
 	// Tendermint logging flags
 	FlagLogLevel  = "log_level"
@@ -131,6 +132,7 @@ func AddPaginationFlagsToCmd(cmd *cobra.Command, query string) {
 	cmd.Flags().Uint64(FlagOffset, 0, fmt.Sprintf("pagination offset of %s to query for", query))
 	cmd.Flags().Uint64(FlagLimit, 100, fmt.Sprintf("pagination limit of %s to query for", query))
 	cmd.Flags().Bool(FlagCountTotal, false, fmt.Sprintf("count total number of records in %s to query for", query))
+	cmd.Flags().Bool(FlagReverse, false, "results are sorted in descending order")
 }
 
 // GasSetting encapsulates the possible values passed through the --gas flag.

client/utils.go

@@ -51,6 +51,7 @@ func ReadPageRequest(flagSet *pflag.FlagSet) (*query.PageRequest, error) {
 	limit, _ := flagSet.GetUint64(flags.FlagLimit)
 	countTotal, _ := flagSet.GetBool(flags.FlagCountTotal)
 	page, _ := flagSet.GetUint64(flags.FlagPage)
+	reverse, _ := flagSet.GetBool(flags.FlagReverse)
 
 	if page > 1 && offset > 0 {
 		return nil, sdkerrors.Wrap(sdkerrors.ErrInvalidRequest, "page and offset cannot be used together")
@@ -65,5 +66,6 @@ func ReadPageRequest(flagSet *pflag.FlagSet) (*query.PageRequest, error) {
 		Offset:     offset,
 		Limit:      limit,
 		CountTotal: countTotal,
+		Reverse:    reverse,
 	}, nil
 }

docs/core/proto-docs.md

@@ -680,6 +680,7 @@ pagination. Ex:
 | `offset` | [uint64](#uint64) |  | offset is a numeric offset that can be used when key is unavailable. It is less efficient than using key. Only one of offset or key should be set. |
 | `limit` | [uint64](#uint64) |  | limit is the total number of results to be returned in the result page. If left empty it will default to a value to be set by each app. |
 | `count_total` | [bool](#bool) |  | count_total is set to true to indicate that the result set should include a count of the total number of items available for pagination in UIs. count_total is only respected when offset is used. It is ignored when key is set. |
+| `reverse` | [bool](#bool) |  | reverse is set to true indicates that, results to be returned in the descending order. |
 
 
 

proto/cosmos/base/query/v1beta1/pagination.proto

@@ -30,6 +30,9 @@ message PageRequest {
   // count_total is only respected when offset is used. It is ignored when key
   // is set.
   bool count_total = 4;
+
+  // reverse is set to true indicates that, results to be returned in the descending order.
+  bool reverse = 5;
 }
 
 // PageResponse is to be embedded in gRPC response messages where the

types/query/filtered_pagination.go

@@ -29,6 +29,7 @@ func FilteredPaginate(
 	key := pageRequest.Key
 	limit := pageRequest.Limit
 	countTotal := pageRequest.CountTotal
+	reverse := pageRequest.Reverse
 
 	if offset > 0 && key != nil {
 		return nil, fmt.Errorf("invalid request, either offset or key is expected, got both")
@@ -42,7 +43,7 @@ func FilteredPaginate(
 	}
 
 	if len(key) != 0 {
-		iterator := prefixStore.Iterator(key, nil)
+		iterator := getIterator(prefixStore, key, reverse)
 		defer iterator.Close()
 
 		var numHits uint64
@@ -73,7 +74,7 @@ func FilteredPaginate(
 		}, nil
 	}
 
-	iterator := prefixStore.Iterator(nil, nil)
+	iterator := getIterator(prefixStore, nil, reverse)
 	defer iterator.Close()
 
 	end := offset + limit

types/query/filtered_pagination_test.go

@@ -89,6 +89,87 @@ func (s *paginationTestSuite) TestFilteredPaginations() {
 	s.Require().LessOrEqual(len(balances), 2)
 }
 
+func (s *paginationTestSuite) TestReverseFilteredPaginations() {
+	app, ctx, appCodec := setupTest()
+
+	var balances sdk.Coins
+	for i := 0; i < numBalances; i++ {
+		denom := fmt.Sprintf("foo%ddenom", i)
+		balances = append(balances, sdk.NewInt64Coin(denom, 100))
+	}
+
+	for i := 0; i < 10; i++ {
+		denom := fmt.Sprintf("test%ddenom", i)
+		balances = append(balances, sdk.NewInt64Coin(denom, 250))
+	}
+
+	balances = balances.Sort()
+	addr1 := sdk.AccAddress([]byte("addr1"))
+	acc1 := app.AccountKeeper.NewAccountWithAddress(ctx, addr1)
+	app.AccountKeeper.SetAccount(ctx, acc1)
+	s.Require().NoError(simapp.FundAccount(app, ctx, addr1, balances))
+	store := ctx.KVStore(app.GetKey(types.StoreKey))
+
+	// verify pagination with limit > total values
+	pageReq := &query.PageRequest{Key: nil, Limit: 5, CountTotal: true, Reverse: true}
+	balns, res, err := execFilterPaginate(store, pageReq, appCodec)
+	s.Require().NoError(err)
+	s.Require().NotNil(res)
+	s.Require().Equal(5, len(balns))
+
+	s.T().Log("verify empty request")
+	balns, res, err = execFilterPaginate(store, nil, appCodec)
+	s.Require().NoError(err)
+	s.Require().NotNil(res)
+	s.Require().Equal(10, len(balns))
+	s.Require().Equal(uint64(10), res.Total)
+	s.Require().Nil(res.NextKey)
+
+	s.T().Log("verify default limit")
+	pageReq = &query.PageRequest{Reverse: true}
+	balns, res, err = execFilterPaginate(store, pageReq, appCodec)
+	s.Require().NoError(err)
+	s.Require().NotNil(res)
+	s.Require().Equal(10, len(balns))
+	s.Require().Equal(uint64(10), res.Total)
+
+	s.T().Log("verify nextKey is returned if there are more results")
+	pageReq = &query.PageRequest{Limit: 2, CountTotal: true, Reverse: true}
+	balns, res, err = execFilterPaginate(store, pageReq, appCodec)
+	s.Require().NoError(err)
+	s.Require().NotNil(res)
+	s.Require().Equal(2, len(balns))
+	s.Require().NotNil(res.NextKey)
+	s.Require().Equal(string(res.NextKey), fmt.Sprintf("test7denom"))
+	s.Require().Equal(uint64(10), res.Total)
+
+	s.T().Log("verify both key and offset can't be given")
+	pageReq = &query.PageRequest{Key: res.NextKey, Limit: 1, Offset: 2, Reverse: true}
+	_, _, err = execFilterPaginate(store, pageReq, appCodec)
+	s.Require().Error(err)
+
+	s.T().Log("use nextKey for query and reverse true")
+	pageReq = &query.PageRequest{Key: res.NextKey, Limit: 2, Reverse: true}
+	balns, res, err = execFilterPaginate(store, pageReq, appCodec)
+	s.Require().NoError(err)
+	s.Require().NotNil(res)
+	s.Require().Equal(2, len(balns))
+	s.Require().NotNil(res.NextKey)
+	s.Require().Equal(string(res.NextKey), fmt.Sprintf("test5denom"))
+
+	s.T().Log("verify last page records, nextKey for query and reverse true")
+	pageReq = &query.PageRequest{Key: res.NextKey, Reverse: true}
+	balns, res, err = execFilterPaginate(store, pageReq, appCodec)
+	s.Require().NoError(err)
+	s.Require().NotNil(res)
+	s.Require().Equal(6, len(balns))
+	s.Require().Nil(res.NextKey)
+
+	s.T().Log("verify Reverse pagination returns valid result")
+	s.Require().Equal(balances[235:241].String(), balns.Sort().String())
+
+}
+
 func ExampleFilteredPaginate() {
 	app, ctx, appCodec := setupTest()
 

types/query/pagination.go

@@ -7,6 +7,7 @@ import (
 	"google.golang.org/grpc/status"
 
 	"github.com/cosmos/cosmos-sdk/store/types"
+	db "github.com/tendermint/tm-db"
 )
 
 // DefaultLimit is the default `limit` for queries
@@ -54,6 +55,7 @@ func Paginate(
 	key := pageRequest.Key
 	limit := pageRequest.Limit
 	countTotal := pageRequest.CountTotal
+	reverse := pageRequest.Reverse
 
 	if offset > 0 && key != nil {
 		return nil, fmt.Errorf("invalid request, either offset or key is expected, got both")
@@ -67,13 +69,14 @@ func Paginate(
 	}
 
 	if len(key) != 0 {
-		iterator := prefixStore.Iterator(key, nil)
+		iterator := getIterator(prefixStore, key, reverse)
 		defer iterator.Close()
 
 		var count uint64
 		var nextKey []byte
 
 		for ; iterator.Valid(); iterator.Next() {
+
 			if count == limit {
 				nextKey = iterator.Key()
 				break
@@ -94,7 +97,7 @@ func Paginate(
 		}, nil
 	}
 
-	iterator := prefixStore.Iterator(nil, nil)
+	iterator := getIterator(prefixStore, nil, reverse)
 	defer iterator.Close()
 
 	end := offset + limit
@@ -132,3 +135,19 @@ func Paginate(
 
 	return res, nil
 }
+
+func getIterator(prefixStore types.KVStore, start []byte, reverse bool) db.Iterator {
+	if reverse {
+		var end []byte
+		if start != nil {
+			itr := prefixStore.Iterator(start, nil)
+			defer itr.Close()
+			if itr.Valid() {
+				itr.Next()
+				end = itr.Key()
+			}
+		}
+		return prefixStore.ReverseIterator(nil, end)
+	}
+	return prefixStore.Iterator(start, nil)
+}