Skip to content

Commit

Permalink
feat: add parse transactions cmd (#59)
Browse files Browse the repository at this point in the history
## Description


## Checklist
- [x] Targeted PR against correct branch.
- [ ] Linked to Github issue with discussion and accepted design OR link to spec that describes this work.
- [ ] Wrote unit tests.  
- [x] Re-reviewed `Files changed` in the Github PR explorer.
  • Loading branch information
MonikaCat authored Apr 28, 2022
1 parent 06f7dbb commit 752076b
Show file tree
Hide file tree
Showing 8 changed files with 137 additions and 10 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
## Unreleased
### Changes
- ([\#61](https://github.com/forbole/juno/pull/61)) Updated v3 migration code to handle database users names with a hyphen
- ([\#59](https://github.com/forbole/juno/pull/59)) Added `parse transactios` command to re-fetch missing or incomplete transactions

## v3.1.1
### Changes
Expand Down
2 changes: 2 additions & 0 deletions cmd/parse/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

parseblocks "github.com/forbole/juno/v3/cmd/parse/blocks"
parsegenesis "github.com/forbole/juno/v3/cmd/parse/genesis"
parsetransactions "github.com/forbole/juno/v3/cmd/parse/transactions"
)

// NewParseCmd returns the Cobra command allowing to parse some chain data without having to re-sync the whole database
Expand All @@ -20,6 +21,7 @@ func NewParseCmd(parseCfg *parsecmdtypes.Config) *cobra.Command {
cmd.AddCommand(
parseblocks.NewBlocksCmd(parseCfg),
parsegenesis.NewGenesisCmd(parseCfg),
parsetransactions.NewTransactionsCmd(parseCfg),
)

return cmd
Expand Down
21 changes: 21 additions & 0 deletions cmd/parse/transactions/cmd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package transactions

import (
"github.com/spf13/cobra"

parsecmdtypes "github.com/forbole/juno/v3/cmd/parse/types"
)

// NewTransactionsCmd returns the Cobra command that allows to fix missing or incomplete transactions
func NewTransactionsCmd(parseConfig *parsecmdtypes.Config) *cobra.Command {
cmd := &cobra.Command{
Use: "transactions",
Short: "Parse things related to transactions",
}

cmd.AddCommand(
newTransactionsCmd(parseConfig),
)

return cmd
}
75 changes: 75 additions & 0 deletions cmd/parse/transactions/transactions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package transactions

import (
"fmt"

parsecmdtypes "github.com/forbole/juno/v3/cmd/parse/types"

"github.com/rs/zerolog/log"

"github.com/spf13/cobra"

"github.com/forbole/juno/v3/parser"
"github.com/forbole/juno/v3/types/config"
)

const (
flagStart = "start"
flagEnd = "end"
)

// newTransactionsCmd returns a Cobra command that allows to fix missing or incomplete transactions in database
func newTransactionsCmd(parseConfig *parsecmdtypes.Config) *cobra.Command {
cmd := &cobra.Command{
Use: "all",
Short: "Parse missing or incomplete transactions",
Long: fmt.Sprintf(`Refetch missing or incomplete transactions and store them inside the database.
You can specify a custom height range by using the %s and %s flags.
`, flagStart, flagEnd),
RunE: func(cmd *cobra.Command, args []string) error {
parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, parseConfig)
if err != nil {
return err
}

workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules)
worker := parser.NewWorker(workerCtx, nil, 0)

// Get the flag values
start, _ := cmd.Flags().GetInt64(flagStart)
end, _ := cmd.Flags().GetInt64(flagEnd)

// Get the start height, default to the config's height; use flagStart if set
startHeight := config.Cfg.Parser.StartHeight
if start > 0 {
startHeight = start
}

// Get the end height, default to the node latest height; use flagEnd if set
endHeight, err := parseCtx.Node.LatestHeight()
if err != nil {
return fmt.Errorf("error while getting chain latest block height: %s", err)
}
if end > 0 {
endHeight = end
}

log.Info().Int64("start height", startHeight).Int64("end height", endHeight).
Msg("getting transactions...")
for k := startHeight; k <= endHeight; k++ {
log.Info().Int64("height", k).Msg("processing transactions...")
err = worker.ProcessTransactions(k)
if err != nil {
return fmt.Errorf("error while re-fetching transactions of height %d: %s", k, err)
}
}

return nil
},
}

cmd.Flags().Int64(flagStart, 0, "Height from which to start fetching missing transactions. If 0, the start height inside the config file will be used instead")
cmd.Flags().Int64(flagEnd, 0, "Height at which to finish fetching missing transactions. If 0, the latest height available inside the node will be used instead")

return cmd
}
21 changes: 19 additions & 2 deletions database/postgresql/postgresql.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,19 @@ func (db *Database) saveTxInsidePartition(tx *types.Tx, partitionId int64) error
sqlStatement := `
INSERT INTO transaction
(hash, height, success, messages, memo, signatures, signer_infos, fee, gas_wanted, gas_used, raw_log, logs, partition_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT DO NOTHING`
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
ON CONFLICT (hash, partition_id) DO UPDATE
SET height = excluded.height,
success = excluded.success,
messages = excluded.messages,
memo = excluded.memo,
signatures = excluded.signatures,
signer_infos = excluded.signer_infos,
fee = excluded.fee,
gas_wanted = excluded.gas_wanted,
gas_used = excluded.gas_used,
raw_log = excluded.raw_log,
logs = excluded.logs`

var sigs = make([]string, len(tx.Signatures))
for index, sig := range tx.Signatures {
Expand Down Expand Up @@ -248,7 +260,12 @@ func (db *Database) SaveMessage(msg *types.Message) error {
func (db *Database) saveMessageInsidePartition(msg *types.Message, partitionID int64) error {
stmt := `
INSERT INTO message(transaction_hash, index, type, value, involved_accounts_addresses, height, partition_id)
VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING`
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (transaction_hash, index, partition_id) DO UPDATE
SET height = excluded.height,
type = excluded.type,
value = excluded.value,
involved_accounts_addresses = excluded.involved_accounts_addresses`

_, err := db.Sql.Exec(stmt, msg.TxHash, msg.Index, msg.Type, msg.Value, pq.Array(msg.Addresses), msg.Height, partitionID)
return err
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -120,5 +120,6 @@ require (

replace (
github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
github.com/rs/zerolog => github.com/rs/zerolog v1.21.0
github.com/tendermint/tendermint => github.com/forbole/tendermint v0.34.13-0.20210820072129-a2a4af55563d
)
10 changes: 2 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -827,10 +827,8 @@ github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U=
github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.23.0/go.mod h1:6c7hFfxPOy7TacJc4Fcdi24/J0NKYGzjG8FWRI916Qo=
github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE=
github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo=
github.com/rs/zerolog v1.21.0 h1:Q3vdXlfLNT+OftyBHsU0Y445MD+8m8axjKgf2si0QcM=
github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
Expand Down Expand Up @@ -953,7 +951,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/zondax/hid v0.9.0 h1:eiT3P6vNxAEVxXMw66eZUAAnU2zD33JBkfG/EnfAKl8=
github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
Expand Down Expand Up @@ -1120,7 +1117,6 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
Expand Down Expand Up @@ -1242,7 +1238,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down Expand Up @@ -1340,7 +1335,6 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
16 changes: 16 additions & 0 deletions parser/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,22 @@ func (w Worker) Process(height int64) error {
return w.ExportBlock(block, events, txs, vals)
}

// ProcessTransactions fetches transactions for a given height and stores them into the database.
// It returns an error if the export process fails.
func (w Worker) ProcessTransactions(height int64) error {
block, err := w.node.Block(height)
if err != nil {
return fmt.Errorf("failed to get block from node: %s", err)
}

txs, err := w.node.Txs(block)
if err != nil {
return fmt.Errorf("failed to get transactions for block: %s", err)
}

return w.ExportTxs(txs)
}

// HandleGenesis accepts a GenesisDoc and calls all the registered genesis handlers
// in the order in which they have been registered.
func (w Worker) HandleGenesis(genesisDoc *tmtypes.GenesisDoc, appState map[string]json.RawMessage) error {
Expand Down

0 comments on commit 752076b

Please sign in to comment.