Add SNICKER support to wallets.

This commit uses the now created PSBTWalletMixin and additionally
creates a SNICKERWalletMixin, and adds a SNICKERReceiver object
to jmclient. A test of the end to end workflow of create and then
co-sign a SNICKER coinjoin as per the draft BIP is in test_snicker.
Additional changes:
updated python-bitcointx dependency to >=1.0.5
Minor refactoring of callbacks in tests and additional redeem
script checks to PSBTWalletMixin.sign_psbt.

Note that this work replaces #403 .

jmbitcoin/jmbitcoin/__init__.py

@@ -2,6 +2,7 @@
 from jmbitcoin.secp256k1_main import *
 from jmbitcoin.secp256k1_transaction import *
 from jmbitcoin.secp256k1_deterministic import *
+from jmbitcoin.snicker import *
 from jmbitcoin.amount import *
 from bitcointx import select_chain_params
 from bitcointx.core import (x, b2x, b2lx, lx, COutPoint, CTxOut, CTxIn,

jmbitcoin/jmbitcoin/secp256k1_ecies.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+from builtins import * # noqa: F401
+from future.utils import native
+import coincurve as secp256k1
+import base64
+import hmac
+import hashlib
+import pyaes
+import os
+import jmbitcoin as btc
+
+ECIES_MAGIC_BYTES = b'BIE1'
+
+class ECIESDecryptionError(Exception):
+    pass
+
+# AES primitives. See BIP-SNICKER for specification.
+def aes_encrypt(key, data, iv):
+    encrypter = pyaes.Encrypter(
+        pyaes.AESModeOfOperationCBC(key, iv=native(iv)))
+    enc_data = encrypter.feed(data)
+    enc_data += encrypter.feed()
+
+    return enc_data
+
+def aes_decrypt(key, data, iv):
+    decrypter = pyaes.Decrypter(
+        pyaes.AESModeOfOperationCBC(key, iv=native(iv)))
+    try:
+        dec_data = decrypter.feed(data)
+        dec_data += decrypter.feed()
+    except ValueError:
+        # note decryption errors can come from PKCS7 padding errors
+        raise ECIESDecryptionError()
+    return dec_data
+
+def ecies_encrypt(message, pubkey):
+    """ Take a privkey in raw byte serialization,
+    and a pubkey serialized in compressed, binary format (33 bytes),
+    and output the shared secret as a 32 byte hash digest output.
+    The exact calculation is:
+    shared_secret = SHA256(privkey * pubkey)
+    .. where * is elliptic curve scalar multiplication.
+    See https://github.com/bitcoin/bitcoin/blob/master/src/secp256k1/src/modules/ecdh/main_impl.h
+    for implementation details.
+    """
+    # create an ephemeral pubkey for this encryption:
+    while True:
+        r = os.urandom(32)
+        # use compressed serialization of the pubkey R:
+        try:
+            R = btc.privkey_to_pubkey(r + b"\x01")
+            break
+        except:
+            # accounts for improbable overflow:
+            continue
+    # note that this is *not* ECDH as in the secp256k1_ecdh module,
+    # since it uses sha512:
+    ecdh_key = btc.multiply(r, pubkey)
+    key = hashlib.sha512(ecdh_key).digest()
+    iv, key_e, key_m = key[0:16], key[16:32], key[32:]
+    ciphertext = aes_encrypt(key_e, message, iv=iv)
+    encrypted = ECIES_MAGIC_BYTES + R + ciphertext
+    mac = hmac.new(key_m, encrypted, hashlib.sha256).digest()
+    return base64.b64encode(encrypted + mac)    
+
+def ecies_decrypt(privkey, encrypted):
+    if len(privkey) == 33 and privkey[-1] == 1:
+        privkey = privkey[:32]
+    encrypted = base64.b64decode(encrypted)
+    if len(encrypted) < 85:
+        raise Exception('invalid ciphertext: length')
+    magic = encrypted[:4]
+    if magic != ECIES_MAGIC_BYTES:
+        raise ECIESDecryptionError()
+    ephemeral_pubkey = encrypted[4:37]
+    try:
+        testR = secp256k1.PublicKey(ephemeral_pubkey)
+    except:
+        raise ECIESDecryptionError()
+    ciphertext = encrypted[37:-32]
+    mac = encrypted[-32:]
+    ecdh_key = btc.multiply(privkey, ephemeral_pubkey)
+    key = hashlib.sha512(ecdh_key).digest()
+    iv, key_e, key_m = key[0:16], key[16:32], key[32:]
+    if mac != hmac.new(key_m, encrypted[:-32], hashlib.sha256).digest():
+        raise ECIESDecryptionError()
+    return aes_decrypt(key_e, ciphertext, iv=iv)    
+

jmbitcoin/jmbitcoin/secp256k1_main.py

@@ -8,7 +8,7 @@
 import coincurve as secp256k1
 
 from bitcointx import base58
-from bitcointx.core import Hash
+from bitcointx.core import Hash, CBitcoinTransaction
 from bitcointx.core.key import CKeyBase, CPubKey
 from bitcointx.signmessage import BitcoinMessage
 
@@ -166,6 +166,18 @@ def add_privkeys(priv1, priv2):
         res += b'\x01'
     return res
 
+def ecdh(privkey, pubkey):
+    """ Take a privkey in raw byte serialization,
+    and a pubkey serialized in compressed, binary format (33 bytes),
+    and output the shared secret as a 32 byte hash digest output.
+    The exact calculation is:
+    shared_secret = SHA256(privkey * pubkey)
+    .. where * is elliptic curve scalar multiplication.
+    See https://github.com/bitcoin/bitcoin/blob/master/src/secp256k1/src/modules/ecdh/main_impl.h
+    for implementation details.
+    """
+    secp_privkey = secp256k1.PrivateKey(privkey)
+    return secp_privkey.ecdh(pubkey)
 
 def ecdsa_raw_sign(msg,
                    priv,

jmbitcoin/jmbitcoin/snicker.py

@@ -0,0 +1,58 @@
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+from builtins import * # noqa: F401
+
+# Implementation of proposal as per
+# https://gist.github.com/AdamISZ/2c13fb5819bd469ca318156e2cf25d79
+# (BIP SNICKER)
+# TODO: BIP69 is removed in this implementation, will update BIP draft.
+
+from jmbitcoin.secp256k1_ecies import *
+from jmbitcoin.secp256k1_main import *
+from jmbitcoin.secp256k1_transaction import *
+
+SNICKER_MAGIC_BYTES = b'SNICKER'
+
+# Flags may be added in future versions
+SNICKER_FLAG_NONE = b"\x00"
+
+def snicker_pubkey_tweak(pub, tweak):
+    """ use secp256k1 library to perform tweak.
+    Both `pub` and `tweak` are expected as byte strings
+    (33 and 32 bytes respectively).
+    Return value is also a 33 byte string serialization
+    of the resulting pubkey (compressed).
+    """
+    base_pub = secp256k1.PublicKey(pub)
+    return base_pub.add(tweak).format()
+
+def snicker_privkey_tweak(priv, tweak):
+    """ use secp256k1 library to perform tweak.
+    Both `priv` and `tweak` are expected as byte strings
+    (32 or 33 and 32 bytes respectively).
+    Return value isa 33 byte string serialization
+    of the resulting private key/secret (with compression flag).
+    """
+    if len(priv) == 33 and priv[-1] == 1:
+        priv = priv[:-1]
+    base_priv = secp256k1.PrivateKey(priv)
+    return base_priv.add(tweak).secret + b'\x01'
+
+def verify_snicker_output(tx, pub, tweak, spk_type='p2sh-p2wpkh'):
+    """ A convenience function to check that one output address in a transaction
+    is a SNICKER-type tweak of an existing key. Returns the index of the output
+    for which this is True (and there must be only 1), and the derived spk,
+    or -1 and None if it is not found exactly once.
+    TODO Add support for other scriptPubKey types.
+    """
+    assert isinstance(tx, btc.CBitcoinTransaction)
+    expected_destination_pub = snicker_pubkey_tweak(pub, tweak)
+    expected_destination_spk = pubkey_to_p2sh_p2wpkh_script(expected_destination_pub)
+    found = 0
+    for i, o in enumerate(tx.vout):
+        if o.scriptPubKey == expected_destination_spk:
+            found += 1
+            found_index = i
+    if found != 1:
+        return -1, None
+    return found_index, expected_destination_spk

jmbitcoin/setup.py

@@ -9,5 +9,6 @@
       author_email='',
       license='GPL',
       packages=['jmbitcoin'],
-      install_requires=['future', 'coincurve', 'python-bitcointx'],
+      install_requires=['future', 'coincurve',
+                        'python-bitcointx>=1.0.5', 'pyaes'],
       zip_safe=False)

jmbitcoin/test/test_ecdh.py

@@ -0,0 +1,60 @@
+#! /usr/bin/env python
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+from builtins import * # noqa: F401
+'''Tests coincurve binding to libsecp256k1 ecdh module code'''
+
+import hashlib
+import jmbitcoin as btc
+from jmbase import hextobin
+import pytest
+import os
+import json
+testdir = os.path.dirname(os.path.realpath(__file__))
+
+def test_ecdh():
+    """Using private key test vectors from Bitcoin Core.
+    1. Import a set of private keys from the json file.
+    2. Calculate the corresponding public keys.
+    3. Do ECDH on the cartesian product (x, Y), with x private
+    and Y public keys, for all combinations.
+    4. Compare the result from CoinCurve with the manual
+    multiplication xY following by hash (sha256). Note that
+    sha256(xY) is the default hashing function used for ECDH
+    in libsecp256k1.
+
+    Since there are about 20 private keys in the json file, this
+    creates around 400 test cases (note xX is still valid).
+    """
+    with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f:
+        json_data = f.read()
+        valid_keys_list = json.loads(json_data)
+        extracted_privkeys = []
+        for a in valid_keys_list:
+            key, hex_key, prop_dict = a
+            if prop_dict["isPrivkey"]:
+                c, k = btc.read_privkey(hextobin(hex_key))
+                extracted_privkeys.append(k)
+    extracted_pubkeys = [btc.privkey_to_pubkey(x) for x in extracted_privkeys]
+    for p in extracted_privkeys:
+        for P in extracted_pubkeys:
+            c, k = btc.read_privkey(p)
+            shared_secret = btc.ecdh(k, P)
+            assert len(shared_secret) == 32
+            # try recreating the shared secret manually:
+            pre_secret = btc.multiply(p, P)
+            derived_secret = hashlib.sha256(pre_secret).digest()
+            assert derived_secret == shared_secret
+
+    # test some important failure cases; null key, overflow case
+    privkeys_invalid = [b'\x00'*32, hextobin(
+        'fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141')]
+    for p in privkeys_invalid:
+        with pytest.raises(Exception) as e_info:
+            shared_secret = btc.ecdh(p, extracted_pubkeys[0])
+    pubkeys_invalid = [b'0xff' + extracted_pubkeys[0][1:], b'0x00'*12]
+    for p in extracted_privkeys:
+        with pytest.raises(Exception) as e_info:
+            shared_secret = btc.ecdh(p, pubkeys_invalid[0])
+        with pytest.raises(Exception) as e_info:
+            shared_secret = btc.ecdh(p, pubkeys_invalid[1])

jmbitcoin/test/test_ecies.py

@@ -0,0 +1,43 @@
+#! /usr/bin/env python
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+from builtins import * # noqa: F401
+'''Tests ECIES implementation as defined in BIP-SNICKER
+(and will be updated if that is).'''
+
+from jmbase import hextobin
+import jmbitcoin as btc
+import base64
+import os
+import json
+testdir = os.path.dirname(os.path.realpath(__file__))
+
+def test_ecies():
+    """Using private key test vectors from Bitcoin Core.
+    1. Import a set of private keys from the json file.
+    2. Calculate the corresponding public keys.
+    3. Do ECDH on the cartesian product (x, Y), with x private
+    and Y public keys, for all combinations.
+    4. Compare the result from CoinCurve with the manual
+    multiplication xY following by hash (sha256). Note that
+    sha256(xY) is the default hashing function used for ECDH
+    in libsecp256k1.
+
+    Since there are about 20 private keys in the json file, this
+    creates around 400 test cases (note xX is still valid).
+    """
+    with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f:
+        json_data = f.read()
+        valid_keys_list = json.loads(json_data)
+        print("got valid keys list")
+        extracted_privkeys = []
+        for a in valid_keys_list:
+            key, hex_key, prop_dict = a
+            if prop_dict["isPrivkey"]:
+                c, k = btc.read_privkey(hextobin(hex_key))
+
+                extracted_privkeys.append(k)
+    extracted_pubkeys = [btc.privkey_to_pubkey(x) for x in extracted_privkeys]
+    for (priv, pub) in zip(extracted_privkeys, extracted_pubkeys):
+        test_message = base64.b64encode(os.urandom(15)*20)
+        assert btc.ecies_decrypt(priv, btc.ecies_encrypt(test_message, pub)) == test_message

jmclient/jmclient/__init__.py

@@ -53,6 +53,7 @@
 from .wallet_service import WalletService
 from .maker import Maker, P2EPMaker
 from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain
+from .snicker_receiver import SNICKERError, SNICKERReceiver
 
 # Set default logging handler to avoid "No handler found" warnings.
 

jmclient/jmclient/cryptoengine.py

@@ -185,7 +185,7 @@ def privkey_to_address(cls, privkey):
     @classmethod
     def pubkey_to_address(cls, pubkey):
         script = cls.pubkey_to_script(pubkey)
-        return str(btc.script_to_address(script, cls.VBYTE))
+        return str(btc.CCoinAddress.from_scriptPubKey(script))
 
     @classmethod
     def pubkey_has_address(cls, pubkey, addr):