Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: GET /transactions: pagination with token always returns a "previous" when moving backward, even on the first page #266

Merged
merged 3 commits into from
Jul 7, 2022
Merged

fix: GET /transactions: pagination with token always returns a "previous" when moving backward, even on the first page #266

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
60f681e
fix: first
Jul 6, 2022
eff6d39
fix: cleanup
Jul 6, 2022
b3ad5a1
fix: simplify Store getTransactions
Jul 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 39 additions & 19 deletions pkg/api/controllers/pagination_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ import (
"go.uber.org/fx"
)

// This test makes sense if maxAdditionalTxs < pageSize
const (
pageSize = 10
maxTxsPages = 3
maxAdditionalTxs = 2
)
Expand All @@ -40,8 +42,6 @@ func getPagination(t *testing.T, api *api.API, txsPages, additionalTxs int) func
return func(ctx context.Context) error {
var rsp *httptest.ResponseRecorder

pageSize := 10

numTxs := txsPages*pageSize + additionalTxs
for i := 0; i < numTxs; i++ {
rsp = internal.PostTransaction(t, api, core.TransactionData{
Expand All @@ -64,7 +64,7 @@ func getPagination(t *testing.T, api *api.API, txsPages, additionalTxs int) func
require.Equal(t, fmt.Sprintf("%d", numTxs), rsp.Header().Get("Count"))

var paginationToken string
var cursor *sharedapi.Cursor[core.Transaction]
cursor := &sharedapi.Cursor[core.Transaction]{}

// MOVING FORWARD
for i := 0; i < txsPages; i++ {
Expand Down Expand Up @@ -111,9 +111,12 @@ func getPagination(t *testing.T, api *api.API, txsPages, additionalTxs int) func
uint64(0), cursor.Data[len(cursor.Data)-1].ID)
}

assert.Empty(t, cursor.Next)

// MOVING BACKWARD
if txsPages > 0 {
for i := 0; i < txsPages; i++ {
back := 0
for cursor.Previous != "" {
paginationToken = cursor.Previous
rsp = internal.GetTransactions(api, url.Values{
"pagination_token": []string{paginationToken},
Expand All @@ -122,6 +125,12 @@ func getPagination(t *testing.T, api *api.API, txsPages, additionalTxs int) func
cursor = internal.DecodeCursorResponse[core.Transaction](t, rsp.Body)
assert.Len(t, cursor.Data, pageSize)
assert.Equal(t, cursor.Next != "", cursor.HasMore)
back++
}
if additionalTxs > 0 {
assert.Equal(t, txsPages, back)
} else {
assert.Equal(t, txsPages-1, back)
}

// First txid of the first page
Expand All @@ -132,6 +141,8 @@ func getPagination(t *testing.T, api *api.API, txsPages, additionalTxs int) func
assert.Equal(t,
uint64((txsPages-1)*pageSize+additionalTxs), cursor.Data[len(cursor.Data)-1].ID)
}

assert.Empty(t, cursor.Previous)
})

t.Run("accounts", func(t *testing.T) {
Expand All @@ -143,11 +154,14 @@ func getPagination(t *testing.T, api *api.API, txsPages, additionalTxs int) func
require.Equal(t, http.StatusOK, rsp.Result().StatusCode)
require.Equal(t, fmt.Sprintf("%d", numAcc), rsp.Header().Get("Count"))

accPages := numAcc / pageSize
additionalAccs := numAcc % pageSize

var paginationToken string
var cursor *sharedapi.Cursor[core.Account]
cursor := &sharedapi.Cursor[core.Account]{}

// MOVING FORWARD
for i := 0; i < txsPages; i++ {
for i := 0; i < accPages; i++ {

values := url.Values{}
if paginationToken == "" {
Expand All @@ -168,34 +182,34 @@ func getPagination(t *testing.T, api *api.API, txsPages, additionalTxs int) func
cursor.Data[0].Address)
} else {
assert.Equal(t,
fmt.Sprintf("accounts:%06d", (txsPages-i)*pageSize+additionalTxs),
fmt.Sprintf("accounts:%06d", (accPages-i)*pageSize+additionalAccs-1),
cursor.Data[0].Address)
}

// Last account of the page
assert.Equal(t,
fmt.Sprintf("accounts:%06d", (txsPages-i-1)*pageSize+additionalTxs+1),
fmt.Sprintf("accounts:%06d", (accPages-i-1)*pageSize+additionalAccs),
cursor.Data[len(cursor.Data)-1].Address)

paginationToken = cursor.Next
}

if additionalTxs > 0 {
if additionalAccs > 0 {
rsp = internal.GetAccounts(api, url.Values{
"pagination_token": []string{paginationToken},
})
assert.Equal(t, http.StatusOK, rsp.Result().StatusCode, rsp.Body.String())
cursor = internal.DecodeCursorResponse[core.Account](t, rsp.Body)
assert.Len(t, cursor.Data, additionalTxs+1)
assert.Len(t, cursor.Data, additionalAccs)
assert.Equal(t, cursor.Next != "", cursor.HasMore)

// First account of the last page
if txsPages == 0 {
if accPages == 0 {
assert.Equal(t, "world",
cursor.Data[0].Address)
} else {
assert.Equal(t,
fmt.Sprintf("accounts:%06d", additionalTxs),
fmt.Sprintf("accounts:%06d", additionalAccs-1),
cursor.Data[0].Address)
}

Expand All @@ -205,22 +219,26 @@ func getPagination(t *testing.T, api *api.API, txsPages, additionalTxs int) func
cursor.Data[len(cursor.Data)-1].Address)
}

assert.Empty(t, cursor.Next)

// MOVING BACKWARD
pageToRead := txsPages - 1
if additionalTxs > 0 {
pageToRead++
}
if pageToRead > 0 { // Only if we have MORE than one page in the result set
for i := 0; i < pageToRead; i++ {
if accPages > 0 {
back := 0
for cursor.Previous != "" {
paginationToken = cursor.Previous
require.NotEmpty(t, paginationToken)
rsp = internal.GetAccounts(api, url.Values{
"pagination_token": []string{paginationToken},
})
assert.Equal(t, http.StatusOK, rsp.Result().StatusCode, rsp.Body.String())
cursor = internal.DecodeCursorResponse[core.Account](t, rsp.Body)
assert.Len(t, cursor.Data, pageSize)
assert.Equal(t, cursor.Next != "", cursor.HasMore)
back++
}
if additionalAccs > 0 {
assert.Equal(t, accPages, back)
} else {
assert.Equal(t, accPages-1, back)
}

// First account of the first page
Expand All @@ -232,6 +250,8 @@ func getPagination(t *testing.T, api *api.API, txsPages, additionalTxs int) func
fmt.Sprintf("accounts:%06d", (txsPages-1)*pageSize+additionalTxs+1),
cursor.Data[len(cursor.Data)-1].Address)
}

assert.Empty(t, cursor.Previous)
})

return nil
Expand Down
26 changes: 13 additions & 13 deletions pkg/storage/sqlstorage/transactions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,13 @@ func (s *Store) getTransactions(ctx context.Context, exec executor, q storage.Tr
}

sb, t := s.buildTransactionsQuery(q)
sb.OrderBy("id desc")
sb.OrderBy("id").Desc()
if q.AfterTxID > 0 {
sb.Where(sb.L("id", q.AfterTxID))
sb.Where(sb.LE("id", q.AfterTxID))
jdupas22 marked this conversation as resolved.
Show resolved Hide resolved
}

// We fetch an additional transaction to know if there are more
sb.Limit(int(q.PageSize + 1))
// We fetch additional transactions to know if there are more before and/or after.
sb.Limit(int(q.PageSize + 2))
t.PageSize = q.PageSize

sqlq, args := sb.BuildWithFlavor(s.schema.Flavor())
Expand All @@ -88,11 +88,7 @@ func (s *Store) getTransactions(ctx context.Context, exec executor, q storage.Tr
}(rows)

for rows.Next() {
var (
ref sql.NullString
ts sql.NullString
)

var ref, ts sql.NullString
tx := core.Transaction{}
if err := rows.Scan(
&tx.ID,
Expand Down Expand Up @@ -121,17 +117,21 @@ func (s *Store) getTransactions(ctx context.Context, exec executor, q storage.Tr
}

var previous, next string
if q.AfterTxID > 0 && len(txs) > 0 {
t.AfterTxID = txs[0].ID + uint64(q.PageSize) + 1

// Page with transactions before
if q.AfterTxID > 0 && len(txs) > 1 && txs[0].ID == q.AfterTxID {
t.AfterTxID = txs[0].ID + uint64(q.PageSize)
txs = txs[1:]
raw, err := json.Marshal(t)
if err != nil {
return sharedapi.Cursor[core.Transaction]{}, s.error(err)
}
previous = base64.RawURLEncoding.EncodeToString(raw)
}

if len(txs) == int(q.PageSize+1) {
txs = txs[:len(txs)-1]
// Page with transactions after
if len(txs) > int(q.PageSize) {
txs = txs[:q.PageSize]
t.AfterTxID = txs[len(txs)-1].ID
raw, err := json.Marshal(t)
if err != nil {
Expand Down