btc: descriptor wallet support

dex/networks/btc: descriptor parsing primitives

This adds descriptor parsing functions and types.
See https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md

client/asset/btc: listdescriptors and get priv key

This adds the RPC code to find the matching private key descriptor
from the listdescriptors response to the address public key in the
getaddressinfo response.

Double check private key accessibility if wallet is unlocked.

harness: make btc descriptor wallets

This makes the beta and gamma wallets into descriptor wallets.

This also removes the BTC test chain archive.

The new-wallet script has an optional $3 arg for the descriptors bool.

.gitignore

@@ -28,3 +28,4 @@ wiki
 dex/testing/loadbot/loadbot
 bin/
 client/webserver/site/template-builder/template-builder
+dex/testing/btc/harnesschain.tar.gz

client/asset/btc/btc.go

@@ -3002,6 +3002,7 @@ func (btc *baseWallet) SignMessage(coin asset.Coin, msg dex.Bytes) (pubkeys, sig
 	if err != nil {
 		return nil, nil, err
 	}
+	defer privKey.Zero()
 	pk := privKey.PubKey()
 	hash := chainhash.HashB(msg) // legacy servers will not accept this signature!
 	sig := ecdsa.Sign(privKey, hash)
@@ -3508,7 +3509,22 @@ func (btc *baseWallet) DepositAddress() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return btc.stringAddr(addr, btc.chainParams)
+	addrStr, err := btc.stringAddr(addr, btc.chainParams)
+	if err != nil {
+		return "", err
+	}
+	if btc.Locked() {
+		return addrStr, nil
+	}
+	// If the wallet is unlocked, be extra cautious and ensure the wallet gave
+	// us an address for which we can retrieve the private keys, regardless of
+	// what ownsAddress would say.
+	priv, err := btc.node.privKeyForAddress(addrStr)
+	if err != nil {
+		return "", fmt.Errorf("private key unavailable for address %v: %w", addrStr, err)
+	}
+	priv.Zero()
+	return addrStr, nil
 }
 
 // NewAddress returns a new address from the wallet. This satisfies the
@@ -4120,6 +4136,7 @@ func (btc *baseWallet) createSig(tx *wire.MsgTx, idx int, pkScript []byte, addr
 	if err != nil {
 		return nil, nil, err
 	}
+	defer privKey.Zero()
 
 	sig, err = btc.signNonSegwit(tx, idx, pkScript, txscript.SigHashAll, privKey, vals, pkScripts)
 	if err != nil {
@@ -4141,6 +4158,7 @@ func (btc *baseWallet) createWitnessSig(tx *wire.MsgTx, idx int, pkScript []byte
 	if err != nil {
 		return nil, nil, err
 	}
+	defer privKey.Zero()
 	sig, err = txscript.RawTxInWitnessSignature(tx, sigHashes, idx, val,
 		pkScript, txscript.SigHashAll, privKey)
 

client/asset/btc/rpcclient.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -41,6 +42,7 @@ const (
 	methodSetTxFee           = "settxfee"
 	methodGetWalletInfo      = "getwalletinfo"
 	methodGetAddressInfo     = "getaddressinfo"
+	methodListDescriptors    = "listdescriptors"
 	methodValidateAddress    = "validateaddress"
 	methodEstimateSmartFee   = "estimatesmartfee"
 	methodSendRawTransaction = "sendrawtransaction"
@@ -97,7 +99,8 @@ type rpcCore struct {
 // RawRequest for wallet-related calls.
 type rpcClient struct {
 	*rpcCore
-	ctx context.Context
+	ctx         context.Context
+	descriptors bool // set on connect like ctx
 }
 
 var _ Wallet = (*rpcClient)(nil)
@@ -126,9 +129,9 @@ func (wc *rpcClient) connect(ctx context.Context, _ *sync.WaitGroup) error {
 	if err != nil {
 		return fmt.Errorf("getwalletinfo failure: %w", err)
 	}
-	if wiRes.Descriptors {
-		return fmt.Errorf("descriptor wallets are not supported, see " +
-			"https://bitcoincore.org/en/releases/0.21.0/#experimental-descriptor-wallets")
+	wc.descriptors = wiRes.Descriptors
+	if wc.descriptors {
+		wc.log.Debug("Using a descriptor wallet.")
 	}
 	return nil
 }
@@ -470,19 +473,176 @@ func (wc *rpcClient) signTx(inTx *wire.MsgTx) (*wire.MsgTx, error) {
 	return outTx, nil
 }
 
+func (wc *rpcClient) listDescriptors(private bool) (*listDescriptorsResult, error) {
+	descriptors := new(listDescriptorsResult)
+	return descriptors, wc.call(methodListDescriptors, anylist{private}, descriptors)
+}
+
 // privKeyForAddress retrieves the private key associated with the specified
 // address.
 func (wc *rpcClient) privKeyForAddress(addr string) (*btcec.PrivateKey, error) {
-	var keyHex string
-	err := wc.call(methodPrivKeyForAddress, anylist{addr}, &keyHex)
+	// Descriptor wallets do not have dumpprivkey.
+	if !wc.descriptors {
+		var keyHex string
+		err := wc.call(methodPrivKeyForAddress, anylist{addr}, &keyHex)
+		if err != nil {
+			return nil, err
+		}
+		wif, err := btcutil.DecodeWIF(keyHex)
+		if err != nil {
+			return nil, err
+		}
+		return wif.PrivKey, nil
+	}
+
+	// With descriptor wallets, we have to get the address' descriptor from
+	// getaddressinfo, parse out its key origin (fingerprint of the master
+	// private key followed by derivation path to the address) and the pubkey of
+	// the address itself. Then we get the private key using listdescriptors
+	// private=true, which returns a set of master private keys and derivation
+	// paths, one of which corresponds to the fingerprint and path from
+	// getaddressinfo. When the parent master private key is identified, we
+	// derive the private key for the address.
+	ai := new(GetAddressInfoResult)
+	if err := wc.call(methodGetAddressInfo, anylist{addr}, ai); err != nil {
+		return nil, fmt.Errorf("getaddressinfo RPC failure: %w", err)
+	}
+	wc.log.Tracef("Address %v descriptor: %v", addr, ai.Descriptor)
+	desc, err := dexbtc.ParseDescriptor(ai.Descriptor)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to parse descriptor %q: %w", ai.Descriptor, err)
+	}
+	if desc.KeyOrigin == nil {
+		return nil, errors.New("address descriptor has no key origin")
+	}
+	// For addresses from imported private keys that have no derivation path in
+	// the key origin, we inspect private keys of type KeyWIFPriv. For addresses
+	// with a derivation path, we match KeyExtended private keys based on the
+	// master key fingerprint and derivation path.
+	fp, addrPath := desc.KeyOrigin.Fingerprint, desc.KeyOrigin.Steps
+	// Should match:
+	//   fp, path = ai.HDMasterFingerprint, ai.HDKeyPath
+	//   addrPath, _, err = dexbtc.ParsePath(path)
+	bareKey := len(addrPath) == 0
+
+	if desc.KeyFmt != dexbtc.KeyHexPub {
+		return nil, fmt.Errorf("not a hexadecimal pubkey: %v", desc.Key)
 	}
-	wif, err := btcutil.DecodeWIF(keyHex)
+	// The key was validated by ParseDescriptor, but check again.
+	addrPubKeyB, err := hex.DecodeString(desc.Key)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("address pubkey not hexadecimal: %w", err)
+	}
+	addrPubKey, err := btcec.ParsePubKey(addrPubKeyB)
+	if err != nil {
+		return nil, fmt.Errorf("invalid pubkey for address: %w", err)
+	}
+	addrPubKeyC := addrPubKey.SerializeCompressed() // may or may not equal addrPubKeyB
+
+	// Get the private key descriptors.
+	masterDescs, err := wc.listDescriptors(true)
+	if err != nil {
+		return nil, fmt.Errorf("listdescriptors RPC failure: %w", err)
+	}
+
+	// We're going to decode a number of private keys that we need to zero.
+	var toClear []interface{ Zero() }
+	defer func() {
+		for _, k := range toClear {
+			k.Zero()
+		}
+	}() // surprisingly, much cleaner than making the loop body below into a function
+	deferZero := func(z interface{ Zero() }) { toClear = append(toClear, z) }
+
+masters:
+	for _, d := range masterDescs.Descriptors {
+		masterDesc, err := dexbtc.ParseDescriptor(d.Descriptor)
+		if err != nil {
+			wc.log.Errorf("Failed to parse descriptor %q: %v", d.Descriptor, err)
+			continue // unexpected, but check the others
+		}
+		if bareKey { // match KeyHexPub -> KeyWIFPriv
+			if masterDesc.KeyFmt != dexbtc.KeyWIFPriv {
+				continue
+			}
+			wif, err := btcutil.DecodeWIF(masterDesc.Key)
+			if err != nil {
+				wc.log.Errorf("Invalid WIF private key: %v", err)
+				continue // ParseDescriptor already validated it, so shouldn't happen
+			}
+			if !bytes.Equal(addrPubKeyC, wif.PrivKey.PubKey().SerializeCompressed()) {
+				continue // not the one
+			}
+			return wif.PrivKey, nil
+		}
+
+		// match KeyHexPub -> [fingerprint/path]KeyExtended
+		if masterDesc.KeyFmt != dexbtc.KeyExtended {
+			continue
+		}
+		// Break the key into its parts and compute the fingerprint of the
+		// master private key.
+		xPriv, fingerprint, pathStr, isRange, err := dexbtc.ParseKeyExtended(masterDesc.Key)
+		if err != nil {
+			wc.log.Debugf("Failed to parse descriptor extended key: %v", err)
+			continue
+		}
+		deferZero(xPriv)
+		if fingerprint != fp {
+			continue
+		}
+		if !xPriv.IsPrivate() { // imported xpub with no private key?
+			wc.log.Debugf("Not an extended private key. Fingerprint: %v", fingerprint)
+			continue
+		}
+		// NOTE: After finding the xprv with the matching fingerprint, we could
+		// skip to checking the private key for a match instead of first
+		// matching the path. Let's just check the path too since fingerprint
+		// collision are possible, and the different address types are allowed
+		// to use descriptors with different fingerprints.
+		if !isRange {
+			continue // imported?
+		}
+		path, _, err := dexbtc.ParsePath(pathStr)
+		if err != nil {
+			wc.log.Debugf("Failed to parse descriptor extended key path %q: %v", pathStr, err)
+			continue
+		}
+		if len(addrPath) != len(path)+1 { // addrPath includes index of self
+			continue
+		}
+		for i := range path {
+			if addrPath[i] != path[i] {
+				continue masters // different path
+			}
+		}
+
+		// NOTE: We could conceivably cache the extended private key for this
+		// address range/branch, but it could be a security risk:
+		// childIdx := addrPath[len(addrPath)-1]
+		// branch, err := dexbtc.DeepChild(xPriv, path)
+		// child, err := branch.Derive(childIdx)
+		child, err := dexbtc.DeepChild(xPriv, addrPath)
+		if err != nil {
+			return nil, fmt.Errorf("address key derivation failed: %v", err) // any point in checking the rest?
+		}
+		deferZero(child)
+		privkey, err := child.ECPrivKey()
+		if err != nil { // only errors if the extended key is not private
+			return nil, err // hdkeychain.ErrNotPrivExtKey
+		}
+		// That's the private key, but do a final check that the pubkey matches
+		// the "pubkey" field of the getaddressinfo response.
+		pubkey := privkey.PubKey().SerializeCompressed()
+		if !bytes.Equal(pubkey, addrPubKeyC) {
+			wc.log.Warnf("Derived wrong pubkey for address %v from matching descriptor %v: %x != %x",
+				addr, d.Descriptor, pubkey, addrPubKey)
+			continue // theoretically could be a fingerprint collision (see KeyOrigin docs)
+		}
+		return privkey, nil
 	}
-	return wif.PrivKey, nil
+
+	return nil, errors.New("no private key found for address")
 }
 
 // getWalletTransaction retrieves the JSON-RPC gettransaction result.
@@ -736,9 +896,9 @@ func (wc *rpcClient) searchBlockForRedemptions(ctx context.Context, reqs map[out
 	return
 }
 
-// call is used internally to marshal parmeters and send requests to the RPC
-// server via (*rpcclient.Client).RawRequest. If `thing` is non-nil, the result
-// will be marshaled into `thing`.
+// call is used internally to marshal parameters and send requests to the RPC
+// server via (*rpcclient.Client).RawRequest. If thing is non-nil, the result
+// will be marshaled into thing.
 func (wc *rpcClient) call(method string, args anylist, thing interface{}) error {
 	params := make([]json.RawMessage, 0, len(args))
 	for i := range args {

client/asset/btc/wallettypes.go

@@ -132,7 +132,35 @@ type GetWalletInfoResult struct {
 	Descriptors bool `json:"descriptors"` // Descriptor wallets that do not support dumpprivkey
 }
 
-// GetAddressInfoResult models the data from the getaddressinfo command.
+// GetAddressInfoResult models some of the data from the getaddressinfo command.
 type GetAddressInfoResult struct {
-	IsMine bool `json:"ismine"`
+	IsMine     bool   `json:"ismine"`
+	Descriptor string `json:"desc"` // e.g. "wpkh([b940190e/84'/1'/0'/0/0]0300034...)#0pfw7rck"
+
+	// The following fields are unused by DEX, but modeled here for completeness
+	// and debugging:
+
+	ParentDesc          string `json:"parent_desc"`         // e.g. "wpkh([b940190e/84'/1'/0']tpubDCo.../0/*)#xn4kr3dw" meaning range of external addresses
+	HDKeyPath           string `json:"hdkeypath"`           // e.g. "m/84'/1'/0'/0/0"
+	HDMasterFingerprint string `json:"hdmasterfingerprint"` // e.g. "b940190e"
+}
+
+type listDescriptorsResult struct {
+	WalletName  string `json:"wallet_name"`
+	Descriptors []*struct {
+		Descriptor string `json:"desc"` // public or private depending on private RPC arg
+
+		// The following fields are unused in this package, but they are modeled
+		// here for completeness and debugging:
+
+		TimeStamp int64 `json:"timestamp"` // creation time
+		// Active makes it the descriptor for the corresponding output
+		// type/externality e.g. wpkh. Must be true for "ranged" descriptors,
+		// which are those for address derivation. Conversely, imported single
+		// private keys are not active.
+		Active   bool    `json:"active"`
+		Internal bool    `json:"internal"` // i.e. change, only set when active
+		Range    []int64 `json:"range"`    // set for ranged descriptors, pertains to gap limit and current index
+		Next     int64   `json:"next"`     // next index to addresses generation; only set for ranged descriptors
+	} `json:"descriptors"`
 }