server/asset: update live tests for p2pk and fix btc bugs

server/asset/doge: add BlockDeserializer

server/asset/btc/btc.go

@@ -130,6 +130,10 @@ type Backend struct {
 	log         dex.Logger
 	decodeAddr  dexbtc.AddressDecoder
 	estimateFee func(*RPCClient) (uint64, error)
+	// booleanGetBlockRPC corresponds to BackendCloneConfig.BooleanGetBlockRPC
+	// field and is used by RPCClient, which is constructed on Connect.
+	booleanGetBlockRPC bool
+	blockDeserializer  func([]byte) (*wire.MsgBlock, error)
 }
 
 // Check that Backend satisfies the Backend interface.
@@ -171,15 +175,17 @@ func newBTC(cloneCfg *BackendCloneConfig, cfg *dexbtc.RPCConfig) *Backend {
 	}
 
 	return &Backend{
-		cfg:         cfg,
-		name:        cloneCfg.Name,
-		blockCache:  newBlockCache(),
-		blockChans:  make(map[chan *asset.BlockUpdate]struct{}),
-		chainParams: cloneCfg.ChainParams,
-		log:         cloneCfg.Logger,
-		segwit:      cloneCfg.Segwit,
-		decodeAddr:  addrDecoder,
-		estimateFee: feeEstimator,
+		cfg:                cfg,
+		name:               cloneCfg.Name,
+		blockCache:         newBlockCache(),
+		blockChans:         make(map[chan *asset.BlockUpdate]struct{}),
+		chainParams:        cloneCfg.ChainParams,
+		log:                cloneCfg.Logger,
+		segwit:             cloneCfg.Segwit,
+		decodeAddr:         addrDecoder,
+		estimateFee:        feeEstimator,
+		booleanGetBlockRPC: cloneCfg.BooleanGetBlockRPC,
+		blockDeserializer:  cloneCfg.BlockDeserializer,
 	}
 }
 
@@ -197,6 +203,11 @@ type BackendCloneConfig struct {
 	// FeeEstimator provides a way to get fees given an RawRequest-enabled
 	// client and a confirmation target.
 	FeeEstimator func(*RPCClient) (uint64, error)
+	// BooleanGetBlockRPC causes the RPC client to use a boolean second argument
+	// for the getblock endpoint, instead of Bitcoin's numeric.
+	BooleanGetBlockRPC bool
+	// BlockDeserializer can be used in place of (*wire.MsgBlock).Deserialize.
+	BlockDeserializer func(blk []byte) (*wire.MsgBlock, error)
 }
 
 // NewBTCClone creates a BTC backend for a set of network parameters and default
@@ -238,8 +249,10 @@ func (btc *Backend) Connect(ctx context.Context) (*sync.WaitGroup, error) {
 	}
 
 	btc.node = &RPCClient{
-		ctx:       ctx,
-		requester: client,
+		ctx:                    ctx,
+		requester:              client,
+		booleanVerboseGetBlock: btc.booleanGetBlockRPC,
+		blockDeserializer:      btc.blockDeserializer,
 	}
 
 	// Prime the cache

server/asset/btc/rpcclient.go

@@ -4,14 +4,17 @@
 package btc
 
 import (
+	"bytes"
 	"context"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"strings"
 
+	"decred.org/dcrdex/dex"
 	"github.com/btcsuite/btcd/btcjson"
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
+	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
 )
 
@@ -23,6 +26,7 @@ const (
 	methodGetRawTransaction = "getrawtransaction"
 	methodGetBlock          = "getblock"
 	methodGetIndexInfo      = "getindexinfo"
+	methodGetBlockHeader    = "getblockheader"
 )
 
 // RawRequester is for sending context-aware RPC requests, and has methods for
@@ -37,8 +41,10 @@ type RawRequester interface {
 // RPCClient is a bitcoind wallet RPC client that uses rpcclient.Client's
 // RawRequest for wallet-related calls.
 type RPCClient struct {
-	ctx       context.Context
-	requester RawRequester
+	ctx                    context.Context
+	requester              RawRequester
+	booleanVerboseGetBlock bool
+	blockDeserializer      func([]byte) (*wire.MsgBlock, error)
 }
 
 func (rc *RPCClient) callHashGetter(method string, args anylist) (*chainhash.Hash, error) {
@@ -128,10 +134,51 @@ func (rc *RPCClient) GetRawTransactionVerbose(txHash *chainhash.Hash) (*btcjson.
 		true}, res)
 }
 
-// GetBlockVerbose fetches verbose block data for the specified hash.
+// getBlock fetches raw block data, and the "verbose" block header, for the
+// block with the given hash. The verbose block header return is separate
+// because it contains other useful info like the height and median time that
+// the wire type does not contain.
+func (rc *RPCClient) getBlock(blockHash *chainhash.Hash) (*wire.MsgBlock, *btcjson.GetBlockHeaderVerboseResult, error) {
+	arg := interface{}(0)
+	if rc.booleanVerboseGetBlock {
+		arg = false
+	}
+	var blockB dex.Bytes // UnmarshalJSON hex -> bytes
+	err := rc.call(methodGetBlock, anylist{blockHash.String(), arg}, &blockB)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var msgBlock *wire.MsgBlock
+	if rc.blockDeserializer == nil {
+		msgBlock = &wire.MsgBlock{}
+		if err := msgBlock.Deserialize(bytes.NewReader(blockB)); err != nil {
+			return nil, nil, err
+		}
+	} else {
+		msgBlock, err = rc.blockDeserializer(blockB)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	verboseHeader := new(btcjson.GetBlockHeaderVerboseResult)
+	err = rc.call(methodGetBlockHeader, anylist{blockHash.String(), true}, verboseHeader)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return msgBlock, verboseHeader, nil
+}
+
+// GetBlockVerbose fetches verbose block data for the block with the given hash.
 func (rc *RPCClient) GetBlockVerbose(blockHash *chainhash.Hash) (*btcjson.GetBlockVerboseResult, error) {
+	arg := interface{}(1)
+	if rc.booleanVerboseGetBlock {
+		arg = true
+	}
 	res := new(btcjson.GetBlockVerboseResult)
-	return res, rc.call(methodGetBlock, anylist{blockHash.String(), true}, res)
+	return res, rc.call(methodGetBlock, anylist{blockHash.String(), arg}, res)
 }
 
 // RawRequest is a wrapper func for callers that are not context-enabled.

server/asset/btc/testing.go

@@ -5,10 +5,12 @@ package btc
 
 import (
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"testing"
 
 	dexbtc "decred.org/dcrdex/dex/networks/btc"
+	"decred.org/dcrdex/server/asset"
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/btcsuite/btcd/txscript"
@@ -207,18 +209,21 @@ out:
 // blockchain literacy. Ideally, the stats will show no scripts which were
 // unparseable by the backend, but the presence of unknowns is not an error.
 func LiveUTXOStats(btc *Backend, t *testing.T) {
-	numToDo := 1000
+	const numToDo = 5000
 	hash, err := btc.node.GetBestBlockHash()
 	if err != nil {
 		t.Fatalf("error getting best block hash: %v", err)
 	}
-	block, err := btc.node.GetBlockVerbose(hash)
+	block, verboseHeader, err := btc.node.getBlock(hash)
 	if err != nil {
 		t.Fatalf("error getting best block verbose: %v", err)
 	}
+	height := verboseHeader.Height
+	t.Logf("Processing block %v (%d)", hash, height)
 	type testStats struct {
 		p2pkh    int
 		p2wpkh   int
+		p2pk     int
 		p2sh     int
 		p2wsh    int
 		zeros    int
@@ -234,24 +239,13 @@ func LiveUTXOStats(btc *Backend, t *testing.T) {
 	var processed int
 out:
 	for {
-		for _, txid := range block.Tx {
-			txHash, err := chainhash.NewHashFromStr(txid)
-			if err != nil {
-				t.Fatalf("error parsing transaction hash from %s: %v", txid, err)
-			}
-			tx, err := btc.node.GetRawTransactionVerbose(txHash)
-			if err != nil {
-				t.Fatalf("error fetching transaction %s: %v", txHash, err)
-			}
-			for vout, txOut := range tx.Vout {
+		for _, msgTx := range block.Transactions {
+			for vout, txOut := range msgTx.TxOut {
 				if txOut.Value == 0 {
 					stats.zeros++
 					continue
 				}
-				pkScript, err := hex.DecodeString(txOut.ScriptPubKey.Hex)
-				if err != nil {
-					t.Fatalf("error decoding script from hex %s: %v", txOut.ScriptPubKey.Hex, err)
-				}
+				pkScript := txOut.PkScript
 				scriptType := dexbtc.ParseScriptType(pkScript, nil)
 				if scriptType == dexbtc.ScriptUnsupported {
 					unknowns = append(unknowns, pkScript)
@@ -262,50 +256,58 @@ out:
 				if processed >= numToDo {
 					break out
 				}
+				txhash := msgTx.TxHash()
 				if scriptType.IsP2PKH() {
-					if scriptType.IsSegwit() {
-						stats.p2wpkh++
-					} else {
-						stats.p2pkh++
-					}
+					stats.p2pkh++
+				} else if scriptType.IsP2WPKH() {
+					stats.p2wpkh++
+				} else if scriptType.IsP2SH() {
+					stats.p2sh++
+					continue // no redeem script, can't use the utxo method
+				} else if scriptType.IsP2WSH() {
+					stats.p2wsh++
+					continue // no redeem script, can't use the utxo method
+				} else if scriptType.IsP2PK() { // rare, so last
+					t.Logf("p2pk: txout %v:%d", txhash, vout)
+					stats.p2pk++
 				} else {
-					if scriptType.IsSegwit() {
-						stats.p2wsh++
-					} else {
-						stats.p2sh++
-					}
-				}
-				if scriptType.IsP2SH() {
-					continue
+					stats.unknown++
+					t.Logf("other unknown script type: %v", scriptType)
 				}
 				stats.checked++
-				utxo, err := btc.utxo(txHash, uint32(vout), nil)
+
+				utxo, err := btc.utxo(&txhash, uint32(vout), nil)
 				if err != nil {
-					stats.utxoErr++
+					if !errors.Is(err, asset.CoinNotFoundError) {
+						t.Log(err, txhash)
+						stats.utxoErr++
+					}
 					continue
 				}
 				stats.feeRates = append(stats.feeRates, utxo.FeeRate())
 				stats.found++
 				stats.utxoVal += utxo.Value()
 			}
 		}
-		prevHash, err := chainhash.NewHashFromStr(block.PreviousHash)
-		if err != nil {
-			t.Fatalf("error decoding previous block hash: %v", err)
-		}
-		block, err = btc.node.GetBlockVerbose(prevHash)
+		prevHash := block.Header.PrevBlock
+		block, verboseHeader, err = btc.node.getBlock(&prevHash)
 		if err != nil {
 			t.Fatalf("error getting previous block verbose: %v", err)
 		}
+		height = verboseHeader.Height
+		h0 := block.BlockHash()
+		hash = &h0
+		t.Logf("Processing block %v (%d)", hash, height)
 	}
 	t.Logf("%d P2PKH scripts", stats.p2pkh)
 	t.Logf("%d P2WPKH scripts", stats.p2wpkh)
+	t.Logf("%d P2PK scripts", stats.p2pk)
 	t.Logf("%d P2SH scripts", stats.p2sh)
 	t.Logf("%d P2WSH scripts", stats.p2wsh)
 	t.Logf("%d zero-valued outputs", stats.zeros)
-	t.Logf("%d P2(W)PKH UTXOs found of %d checked, %.1f%%", stats.found, stats.checked, float64(stats.found)/float64(stats.checked)*100)
+	t.Logf("%d P2(W)PK(H) UTXOs found of %d checked, %.1f%%", stats.found, stats.checked, float64(stats.found)/float64(stats.checked)*100)
 	t.Logf("total unspent value counted: %.2f", float64(stats.utxoVal)/1e8)
-	t.Logf("%d P2PKH UTXO retrieval errors (likely already spent, OK)", stats.utxoErr)
+	t.Logf("%d P2PK(H) UTXO retrieval errors", stats.utxoErr)
 	numUnknown := len(unknowns)
 	if numUnknown > 0 {
 		numToShow := 5

server/asset/btc/utxo.go

@@ -287,7 +287,7 @@ func (utxo *UTXO) Confirmations(context.Context) (int64, error) {
 		// See if we can find the utxo in another block.
 		newUtxo, err := utxo.btc.utxo(&utxo.tx.hash, utxo.vout, utxo.redeemScript)
 		if err != nil {
-			return -1, fmt.Errorf("utxo block is not mainchain")
+			return -1, fmt.Errorf("utxo error: %w", err)
 		}
 		*utxo = *newUtxo
 		return utxo.Confirmations(context.Background())

server/asset/dcr/live_test.go

@@ -51,11 +51,13 @@ func TestMain(m *testing.M) {
 
 		var cancel context.CancelFunc
 		ctx, cancel = context.WithCancel(context.Background())
-		wg := new(sync.WaitGroup)
+		var wg *sync.WaitGroup
 		defer func() {
 			logger.Infof("Shutting down...")
 			cancel()
-			wg.Wait()
+			if wg != nil {
+				wg.Wait()
+			}
 			logger.Infof("done.")
 		}()
 
@@ -87,7 +89,10 @@ func TestLiveUTXO(t *testing.T) {
 	var txs []*wire.MsgTx
 	type testStats struct {
 		p2pkh          int
+		p2pk           int
 		sp2pkh         int
+		p2pkSchnorr    int
+		p2pkEdwards    int
 		p2pkhSchnorr   int
 		p2pkhEdwards   int
 		p2sh           int
@@ -174,7 +179,7 @@ func TestLiveUTXO(t *testing.T) {
 						stats.p2sh++
 					}
 					continue
-				} else if scriptType&dexdcr.ScriptP2PKH != 0 {
+				} else if scriptType.IsP2PKH() {
 					switch {
 					case scriptType&dexdcr.ScriptSigEdwards != 0:
 						stats.p2pkhEdwards++
@@ -185,6 +190,15 @@ func TestLiveUTXO(t *testing.T) {
 					default:
 						stats.p2pkh++
 					}
+				} else if scriptType.IsP2PK() {
+					switch {
+					case scriptType&dexdcr.ScriptSigEdwards != 0:
+						stats.p2pkEdwards++
+					case scriptType&dexdcr.ScriptSigSchnorr != 0:
+						stats.p2pkSchnorr++
+					default:
+						stats.p2pk++
+					}
 				}
 				// Check if its an acceptable script type.
 				scriptTypeOK := scriptType != dexdcr.ScriptUnsupported
@@ -310,6 +324,9 @@ func TestLiveUTXO(t *testing.T) {
 	t.Logf("%d Schnorr P2PKH scripts", stats.p2pkhSchnorr)
 	t.Logf("%d Edwards P2PKH scripts", stats.p2pkhEdwards)
 	t.Logf("%d P2SH scripts", stats.p2sh)
+	t.Logf("%d P2PK scripts", stats.p2pk)
+	t.Logf("%d Schnorr P2PK scripts", stats.p2pkSchnorr)
+	t.Logf("%d Edwards P2PK scripts", stats.p2pkEdwards)
 	t.Logf("%d stake P2SH scripts", stats.sp2sh)
 	t.Logf("%d immature transactions in the last %d blocks", stats.immatureBefore, maturity)
 	t.Logf("%d immature transactions before %d blocks ago", stats.immatureAfter, maturity)

server/asset/doge/doge.go

@@ -109,5 +109,7 @@ func NewBackend(configPath string, logger dex.Logger, network dex.Network) (asse
 
 			return uint64(math.Round(r * 1e5)), nil
 		},
+		BooleanGetBlockRPC: true,
+		BlockDeserializer:  dexdoge.DeserializeBlock,
 	})
 }