Main patch tm-v0.34.13 (#364)

* e2e: avoid starting nodes from the future (#6835) (#6838)

(cherry picked from commit a4cc8317da47ba92d1a97460a953d0e687a46cfb)

Co-authored-by: Sam Kleinman <garen@tychoish.com>

* e2e: cleanup node start function (#6842) (#6848)

I realized after my last commit that my change made a following line of code a bit redundant.

(alternatively my last change was redunadnt to the existing code.)

I took this oppertunity to make some minor cleanups and logging changes to the node changes which I hope will make tests a bit more clear.

(cherry picked from commit a374f74f7c914a3c7640694ae80c33bf74e05688)

Co-authored-by: Sam Kleinman <garen@tychoish.com>

* internal/consensus: update error log (#6863) (#6867)

Issues reported in Osmosis, where the message is extremely long. Also, there is absolutely no reason to log the message IMO. If we must, we can make the message log DEBUG.

(cherry picked from commit 58a6cfff9a3e370e1bb5eabfd86091e0b1186a22)

Co-authored-by: Aleksandr Bezobchuk <alexanderbez@users.noreply.github.com>

* abci: change client to use multi-reader mutexes (backport #6306) (#6873)

* statesync: improve stateprovider handling in the syncer (backport) (#6881)

Co-authored-by: Callum Waters <cmwaters19@gmail.com>

* Fix codecov: `statesync: improve stateprovider handling in the syncer (backport) (#6881)`

* light: fix early erroring (#6905)

Co-authored-by: Callum Waters <cmwaters19@gmail.com>

* Backport the psql indexer into v0.34.x (#6906)

This change backports the PostgreSQL indexing sink, addressing part of #6828.

Development on the main branch has diverged substantially since the v0.34.x
release. It includes package moves, breaking API and protobuf schema changes,
and new APIs, all of which together have a large footprint on the mapping
between the implementation at tip and the v0.34 release branch.

To avoid the need to retrofit all of those improvements, this change works by
injecting the new indexing sink into the existing (v0.34) indexing interfaces
by delegation. This means the backport does _not_ pull in all the newer APIs
for event handling, and thus has minimal impact on existing code written
against the v0.34 package structure.

This change includes the test for the `psql` implementation, and thus updates
some Go module dependencies. Because it does not interact with any other types,
however, I did not add any unit tests to other packages in this change.

Related changes:
 * Update module dependencies for psql backport.
 * Update test data to be type-compatible with the old protobuf types.
 * Add config settings for the PostgreSQL indexer.
 * Clean up some linter settings.
 * Hook up the psql indexer in the node main.

Co-authored-by: M. J. Fromberger <fromberger@interchain.io>

* Fix codecov: `Backport the psql indexer into v0.34.x (#6906)`

Co-authored-by: Sam Kleinman <garen@tychoish.com>
Co-authored-by: Aleksandr Bezobchuk <alexanderbez@users.noreply.github.com>
Co-authored-by: Callum Waters <cmwaters19@gmail.com>
Co-authored-by: M. J. Fromberger <fromberger@interchain.io>

.golangci.yml

@@ -6,6 +6,7 @@ linters:
     - dogsled
     - dupl
     - errcheck
+    - exportloopref
     # - funlen
     # - gochecknoglobals
     # - gochecknoinits
@@ -26,7 +27,7 @@ linters:
     # - maligned
     - nakedret
     - prealloc
-    - scopelint
+    # - scopelint
     - staticcheck
     - structcheck
     - stylecheck

abci/client/client.go

@@ -87,7 +87,7 @@ type ReqRes struct {
 	*types.Request
 	*types.Response // Not set atomically, so be sure to use WaitGroup.
 
-	mtx  tmsync.Mutex
+	mtx  tmsync.RWMutex
 	wg   *sync.WaitGroup
 	done bool             // Gets set to true once *after* WaitGroup.Done().
 	cb   ResponseCallback // A single callback that may be set.
@@ -107,24 +107,24 @@ func NewReqRes(req *types.Request, cb ResponseCallback) *ReqRes {
 // InvokeCallback invokes a thread-safe execution of the configured callback
 // if non-nil.
 func (reqRes *ReqRes) InvokeCallback() {
-	reqRes.mtx.Lock()
-	defer reqRes.mtx.Unlock()
+	reqRes.mtx.RLock()
+	defer reqRes.mtx.RUnlock()
 
 	if reqRes.cb != nil {
 		reqRes.cb(reqRes.Response)
 	}
 }
 
 func (reqRes *ReqRes) SetDone(res *types.Response) (set bool) {
-	reqRes.mtx.Lock()
+	reqRes.mtx.RLock()
 	// TODO should we panic if it's already done?
 	set = !reqRes.done
 	if set {
 		reqRes.Response = res
 		reqRes.done = true
 		reqRes.wg.Done()
 	}
-	reqRes.mtx.Unlock()
+	reqRes.mtx.RUnlock()
 
 	// NOTE `reqRes.cb` is immutable so we're safe to access it at here without `mtx`
 	if set && reqRes.cb != nil {

abci/client/grpc_client.go

@@ -26,11 +26,11 @@ type grpcClient struct {
 	client types.ABCIApplicationClient
 	conn   *grpc.ClientConn
 
-	mtx  tmsync.Mutex
+	mtx  tmsync.RWMutex
 	addr string
 	err  error
 
-	globalCbMtx sync.Mutex
+	globalCbMtx sync.RWMutex
 	globalCb    func(*types.Request, *types.Response) // listens to all callbacks
 }
 
@@ -109,21 +109,21 @@ func (cli *grpcClient) StopForError(err error) {
 }
 
 func (cli *grpcClient) Error() error {
-	cli.mtx.Lock()
-	defer cli.mtx.Unlock()
+	cli.mtx.RLock()
+	defer cli.mtx.RUnlock()
 	return cli.err
 }
 
 func (cli *grpcClient) SetGlobalCallback(globalCb GlobalCallback) {
 	cli.globalCbMtx.Lock()
+	defer cli.globalCbMtx.Unlock()
 	cli.globalCb = globalCb
-	cli.globalCbMtx.Unlock()
 }
 
 func (cli *grpcClient) GetGlobalCallback() (cb GlobalCallback) {
 	cli.globalCbMtx.Lock()
+	defer cli.globalCbMtx.Unlock()
 	cb = cli.globalCb
-	cli.globalCbMtx.Unlock()
 	return cb
 }
 

abci/client/local_client.go

@@ -16,36 +16,44 @@ type localClient struct {
 	service.BaseService
 
 	// TODO: remove `mtx` to increase concurrency. We could remove it because the app should protect itself.
-	mtx *tmsync.Mutex
+	mtx *tmsync.RWMutex
 	// CONTRACT: The application should protect itself from concurrency as an abci server.
 	types.Application
 
-	globalCbMtx tmsync.Mutex
+	globalCbMtx tmsync.RWMutex
 	globalCb    GlobalCallback
 }
 
-func NewLocalClient(mtx *tmsync.Mutex, app types.Application) Client {
+var _ Client = (*localClient)(nil)
+
+// NewLocalClient creates a local client, which will be directly calling the
+// methods of the given app.
+//
+// Both Async and Sync methods ignore the given context.Context parameter.
+func NewLocalClient(mtx *tmsync.RWMutex, app types.Application) Client {
 	if mtx == nil {
-		mtx = new(tmsync.Mutex)
+		mtx = &tmsync.RWMutex{}
 	}
+
 	cli := &localClient{
 		mtx:         mtx,
 		Application: app,
 	}
+
 	cli.BaseService = *service.NewBaseService(nil, "localClient", cli)
 	return cli
 }
 
 func (app *localClient) SetGlobalCallback(globalCb GlobalCallback) {
 	app.globalCbMtx.Lock()
+	defer app.globalCbMtx.Unlock()
 	app.globalCb = globalCb
-	app.globalCbMtx.Unlock()
 }
 
 func (app *localClient) GetGlobalCallback() (cb GlobalCallback) {
 	app.globalCbMtx.Lock()
+	defer app.globalCbMtx.Unlock()
 	cb = app.globalCb
-	app.globalCbMtx.Unlock()
 	return cb
 }
 
@@ -70,8 +78,8 @@ func (app *localClient) EchoAsync(msg string, cb ResponseCallback) *ReqRes {
 }
 
 func (app *localClient) InfoAsync(req types.RequestInfo, cb ResponseCallback) *ReqRes {
-	app.mtx.Lock()
-	defer app.mtx.Unlock()
+	app.mtx.RLock()
+	defer app.mtx.RUnlock()
 
 	reqRes := NewReqRes(types.ToRequestInfo(req), cb)
 	res := app.Application.Info(req)
@@ -112,8 +120,8 @@ func (app *localClient) CheckTxAsync(req types.RequestCheckTx, cb ResponseCallba
 }
 
 func (app *localClient) QueryAsync(req types.RequestQuery, cb ResponseCallback) *ReqRes {
-	app.mtx.Lock()
-	defer app.mtx.Unlock()
+	app.mtx.RLock()
+	defer app.mtx.RUnlock()
 
 	reqRes := NewReqRes(types.ToRequestQuery(req), cb)
 	res := app.Application.Query(req)
@@ -226,8 +234,8 @@ func (app *localClient) EchoSync(msg string) (*types.ResponseEcho, error) {
 }
 
 func (app *localClient) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) {
-	app.mtx.Lock()
-	defer app.mtx.Unlock()
+	app.mtx.RLock()
+	defer app.mtx.RUnlock()
 
 	res := app.Application.Info(req)
 	return &res, nil
@@ -259,8 +267,8 @@ func (app *localClient) CheckTxSync(req types.RequestCheckTx) (*types.ResponseCh
 }
 
 func (app *localClient) QuerySync(req types.RequestQuery) (*types.ResponseQuery, error) {
-	app.mtx.Lock()
-	defer app.mtx.Unlock()
+	app.mtx.RLock()
+	defer app.mtx.RUnlock()
 
 	res := app.Application.Query(req)
 	return &res, nil

abci/client/socket_client.go

@@ -34,11 +34,11 @@ type socketClient struct {
 	reqQueue   chan *ReqRes
 	flushTimer *timer.ThrottleTimer
 
-	mtx     tmsync.Mutex
+	mtx     tmsync.RWMutex
 	err     error
 	reqSent *list.List // list of requests sent, waiting for response
 
-	globalCbMtx tmsync.Mutex
+	globalCbMtx tmsync.RWMutex
 	globalCb    GlobalCallback
 }
 
@@ -101,21 +101,21 @@ func (cli *socketClient) OnStop() {
 
 // Error returns an error if the client was stopped abruptly.
 func (cli *socketClient) Error() error {
-	cli.mtx.Lock()
-	defer cli.mtx.Unlock()
+	cli.mtx.RLock()
+	defer cli.mtx.RUnlock()
 	return cli.err
 }
 
 func (cli *socketClient) SetGlobalCallback(globalCb GlobalCallback) {
 	cli.globalCbMtx.Lock()
+	defer cli.globalCbMtx.Unlock()
 	cli.globalCb = globalCb
-	cli.globalCbMtx.Unlock()
 }
 
 func (cli *socketClient) GetGlobalCallback() (cb GlobalCallback) {
 	cli.globalCbMtx.Lock()
+	defer cli.globalCbMtx.Unlock()
 	cb = cli.globalCb
-	cli.globalCbMtx.Unlock()
 	return cb
 }
 

config/config.go

@@ -1095,7 +1095,12 @@ type TxIndexConfig struct {
 	//   1) "null"
 	//   2) "kv" (default) - the simplest possible indexer,
 	//      backed by key-value storage (defaults to levelDB; see DBBackend).
+	//   3) "psql" - the indexer services backed by PostgreSQL.
 	Indexer string `mapstructure:"indexer"`
+
+	// The PostgreSQL connection configuration, the connection format:
+	// postgresql://<user>:<password>@<host>:<port>/<db>?<opts>
+	PsqlConn string `mapstructure:"psql-conn"`
 }
 
 // DefaultTxIndexConfig returns a default configuration for the transaction indexer.

consensus/byzantine_test.go

@@ -64,7 +64,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
 		blockStore := store.NewBlockStore(blockDB)
 
 		// one for mempool, one for consensus
-		mtx := new(tmsync.Mutex)
+		mtx := new(tmsync.RWMutex)
 		proxyAppConnMem := abcicli.NewLocalClient(mtx, app)
 		proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
 

consensus/common_test.go

@@ -434,7 +434,7 @@ func newStateWithConfigAndBlockStoreWithLoggers(
 	blockStore := store.NewBlockStore(blockDB)
 
 	// one for mempool, one for consensus
-	mtx := new(tmsync.Mutex)
+	mtx := new(tmsync.RWMutex)
 	proxyAppConnMem := abcicli.NewLocalClient(mtx, app)
 	proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
 

consensus/reactor_test.go

@@ -152,7 +152,7 @@ func TestReactorWithEvidence(t *testing.T) {
 		blockStore := store.NewBlockStore(blockDB)
 
 		// one for mempool, one for consensus
-		mtx := new(tmsync.Mutex)
+		mtx := new(tmsync.RWMutex)
 		proxyAppConnMem := abcicli.NewLocalClient(mtx, app)
 		proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
 

consensus/state.go

@@ -916,8 +916,8 @@ func (cs *State) handleMsg(mi msgInfo) {
 			"height", cs.Height,
 			"round", cs.Round,
 			"peer", peerID,
+			"msg_type", fmt.Sprintf("%T", msg),
 			"err", err,
-			"msg", msg,
 		)
 	}
 }

go.mod

@@ -3,45 +3,49 @@ module github.com/line/ostracon
 go 1.15
 
 require (
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/BurntSushi/toml v0.3.1
 	github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d
+	github.com/Microsoft/go-winio v0.5.0 // indirect
 	github.com/Workiva/go-datastructures v1.0.52
+	github.com/adlio/schema v1.1.13
 	github.com/btcsuite/btcd v0.21.0-beta
 	github.com/btcsuite/btcutil v1.0.2
 	github.com/confio/ics23/go v0.6.3
 	github.com/coniks-sys/coniks-go v0.0.0-20180722014011-11acf4819b71
+	github.com/containerd/continuity v0.1.0 // indirect
 	github.com/fortytw2/leaktest v1.3.0
 	github.com/go-kit/kit v0.10.0
 	github.com/go-logfmt/logfmt v0.5.0
 	github.com/gogo/protobuf v1.3.2
-	github.com/golang/protobuf v1.4.3
+	github.com/golang/protobuf v1.5.0
 	github.com/google/orderedcode v0.0.1
 	github.com/gorilla/websocket v1.4.2
 	github.com/gtank/merlin v0.1.1
 	github.com/herumi/bls-eth-go-binary v0.0.0-20200923072303-32b29e5d8cbf
+	github.com/lib/pq v1.2.0
 	github.com/libp2p/go-buffer-pool v0.0.2
 	github.com/line/tm-db/v2 v2.0.0-init.1.0.20210824011847-fcfa67dd3c70
 	github.com/minio/highwayhash v1.0.1
-	github.com/pelletier/go-toml v1.6.0 // indirect
+	github.com/opencontainers/runc v1.0.2 // indirect
+	github.com/ory/dockertest v3.3.5+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.8.0
 	github.com/r2ishiguro/vrf v0.0.0-20180716233122-192de52975eb
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0
 	github.com/rs/cors v1.7.0
 	github.com/sasha-s/go-deadlock v0.2.1-0.20190427202633-1595213edefa
 	github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa
-	github.com/spf13/afero v1.2.2 // indirect
 	github.com/spf13/cobra v1.1.1
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.7.1
 	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/stretchr/testify v1.7.0
 	github.com/tendermint/go-amino v0.16.0
 	github.com/yahoo/coname v0.0.0-20170609175141-84592ddf8673 // indirect
 	golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9
-	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
-	gonum.org/v1/gonum v0.9.3
+	golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f
+	golang.org/x/sys v0.0.0-20210903071746-97244b99971b // indirect
+	gonum.org/v1/gonum v0.8.2
 	google.golang.org/grpc v1.37.0
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )