bech32 destination addresses and script conversions

jmclient validate_address update for bech32 and test
add_tx_notify supports/handles bech32 outputs
bugfix Qt for bech32 address validation

jmbitcoin/jmbitcoin/__init__.py

@@ -4,4 +4,5 @@ from jmbitcoin.secp256k1_transaction import *
 from jmbitcoin.secp256k1_deterministic import *
 from jmbitcoin.bci import *
 from jmbitcoin.btscript import *
+from jmbitcoin.bech32 import *
 

jmbitcoin/jmbitcoin/bech32.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2017 Pieter Wuille
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+"""Reference implementation for Bech32 and segwit addresses."""
+
+
+CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l"
+
+
+def bech32_polymod(values):
+    """Internal function that computes the Bech32 checksum."""
+    generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3]
+    chk = 1
+    for value in values:
+        top = chk >> 25
+        chk = (chk & 0x1ffffff) << 5 ^ value
+        for i in range(5):
+            chk ^= generator[i] if ((top >> i) & 1) else 0
+    return chk
+
+
+def bech32_hrp_expand(hrp):
+    """Expand the HRP into values for checksum computation."""
+    return [ord(x) >> 5 for x in hrp] + [0] + [ord(x) & 31 for x in hrp]
+
+
+def bech32_verify_checksum(hrp, data):
+    """Verify a checksum given HRP and converted data characters."""
+    return bech32_polymod(bech32_hrp_expand(hrp) + data) == 1
+
+
+def bech32_create_checksum(hrp, data):
+    """Compute the checksum values given HRP and data."""
+    values = bech32_hrp_expand(hrp) + data
+    polymod = bech32_polymod(values + [0, 0, 0, 0, 0, 0]) ^ 1
+    return [(polymod >> 5 * (5 - i)) & 31 for i in range(6)]
+
+
+def bech32_encode(hrp, data):
+    """Compute a Bech32 string given HRP and data values."""
+    combined = data + bech32_create_checksum(hrp, data)
+    return hrp + '1' + ''.join([CHARSET[d] for d in combined])
+
+
+def bech32_decode(bech):
+    """Validate a Bech32 string, and determine HRP and data."""
+    if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or
+            (bech.lower() != bech and bech.upper() != bech)):
+        return (None, None)
+    bech = bech.lower()
+    pos = bech.rfind('1')
+    if pos < 1 or pos + 7 > len(bech) or len(bech) > 90:
+        return (None, None)
+    if not all(x in CHARSET for x in bech[pos+1:]):
+        return (None, None)
+    hrp = bech[:pos]
+    data = [CHARSET.find(x) for x in bech[pos+1:]]
+    if not bech32_verify_checksum(hrp, data):
+        return (None, None)
+    return (hrp, data[:-6])
+
+
+def convertbits(data, frombits, tobits, pad=True):
+    """General power-of-2 base conversion."""
+    acc = 0
+    bits = 0
+    ret = []
+    maxv = (1 << tobits) - 1
+    max_acc = (1 << (frombits + tobits - 1)) - 1
+    for value in data:
+        if value < 0 or (value >> frombits):
+            return None
+        acc = ((acc << frombits) | value) & max_acc
+        bits += frombits
+        while bits >= tobits:
+            bits -= tobits
+            ret.append((acc >> bits) & maxv)
+    if pad:
+        if bits:
+            ret.append((acc << (tobits - bits)) & maxv)
+    elif bits >= frombits or ((acc << (tobits - bits)) & maxv):
+        return None
+    return ret
+
+
+def bech32addr_decode(hrp, addr):
+    """Decode a segwit address."""
+    hrpgot, data = bech32_decode(addr)
+    if hrpgot != hrp:
+        return (None, None)
+    decoded = convertbits(data[1:], 5, 8, False)
+    if decoded is None or len(decoded) < 2 or len(decoded) > 40:
+        return (None, None)
+    if data[0] > 16:
+        return (None, None)
+    if data[0] == 0 and len(decoded) != 20 and len(decoded) != 32:
+        return (None, None)
+    return (data[0], decoded)
+
+
+def bech32addr_encode(hrp, witver, witprog):
+    """Encode a segwit address."""
+    ret = bech32_encode(hrp, [witver] + convertbits(witprog, 8, 5))
+    if bech32addr_decode(hrp, ret) == (None, None):
+        return None
+    return ret

jmbitcoin/jmbitcoin/secp256k1_transaction.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python
 import binascii, re, json, copy, sys
 from jmbitcoin.secp256k1_main import *
+from jmbitcoin.bech32 import *
 from _functools import reduce
 import os
 
@@ -333,10 +334,26 @@ def mk_pubkey_script(addr):
 def mk_scripthash_script(addr):
     return 'a914' + b58check_to_hex(addr) + '87'
 
+def segwit_scriptpubkey(witver, witprog):
+    """Construct a Segwit scriptPubKey for a given witness program."""
+    if sys.version_info >= (3, 0):
+        x = bytes([witver + 0x50 if witver else 0, len(witprog)] + witprog)
+    else:
+        x = chr(witver + 0x50) if witver else '\x00'
+        x += chr(len(witprog))
+        x += bytearray(witprog)
+    return x
+
+def mk_native_segwit_script(addr):
+    hrp = addr[:2]
+    ver, prog =  bech32addr_decode(hrp, addr)
+    scriptpubkey = segwit_scriptpubkey(ver, prog)
+    return binascii.hexlify(scriptpubkey)
 # Address representation to output script
 
-
 def address_to_script(addr):
+    if addr[:2] in ['bc', 'tb']:
+        return mk_native_segwit_script(addr)
     if addr[0] == '3' or addr[0] == '2':
         return mk_scripthash_script(addr)
     else:
@@ -350,9 +367,23 @@ def is_p2pkh_script(script):
         return True
     return False
 
-def script_to_address(script, vbyte=0):
+def is_segwit_native_script(script):
+    """Is scriptPubkey of form P2WPKH or P2WSH"""
+    if script[:2] in [b'\x00\x14', b'\x00\x20']:
+        return True
+    return False
+
+def script_to_address(script, vbyte=0, witver=0):
     if re.match('^[0-9a-fA-F]*$', script):
         script = binascii.unhexlify(script)
+    if is_segwit_native_script(script):
+        #hrp interpreted from the vbyte entry, TODO this should be cleaner.
+        if vbyte in [0, 5]:
+            hrp = 'bc'
+        else:
+            hrp = 'tb'
+        return bech32addr_encode(hrp=hrp, witver=witver,
+                                 witprog=[ord(x) for x in script[2:]])
     if is_p2pkh_script(script):
         return bin_to_b58check(script[3:-2], vbyte)  # pubkey hash addresses
     else:

jmbitcoin/test/test_bech32.py

@@ -0,0 +1,143 @@
+#!/usr/bin/python
+from __future__ import print_function
+
+# Copyright (c) 2017 Pieter Wuille
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+"""Reference tests for segwit adresses"""
+
+import sys
+import binascii
+import unittest
+import jmbitcoin as btc
+
+def segwit_scriptpubkey(witver, witprog):
+    """Construct a Segwit scriptPubKey for a given witness program."""
+    if sys.version_info >= (3, 0):
+        x = bytes([witver + 0x50 if witver else 0, len(witprog)] + witprog)
+    else:
+        x = chr(witver + 0x50) if witver else '\x00'
+        x += chr(len(witprog))
+        x += bytearray(witprog)
+    return x
+
+VALID_CHECKSUM = [
+    "A12UEL5L",
+    "an83characterlonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1tt5tgs",
+    "abcdef1qpzry9x8gf2tvdw0s3jn54khce6mua7lmqqqxw",
+    "11qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqc8247j",
+    "split1checkupstagehandshakeupstreamerranterredcaperred2y9e3w",
+]
+
+INVALID_CHECKSUM = [
+    " 1nwldj5",
+    "\x7F" + "1axkwrx",
+    "an84characterslonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1569pvx",
+    "pzry9x0s0muk",
+    "1pzry9x0s0muk",
+    "x1b4n0q5v",
+    "li1dgmt3",
+    "de1lg7wt\xff",
+]
+
+VALID_ADDRESS = [
+    ["BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4", "0014751e76e8199196d454941c45d1b3a323f1433bd6"],
+    ["tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7",
+     "00201863143c14c5166804bd19203356da136c985678cd4d27a1b8c6329604903262"],
+    ["bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx",
+     "5128751e76e8199196d454941c45d1b3a323f1433bd6751e76e8199196d454941c45d1b3a323f1433bd6"],
+    ["BC1SW50QA3JX3S", "6002751e"],
+    ["bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj", "5210751e76e8199196d454941c45d1b3a323"],
+    ["tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy",
+     "0020000000c4a5cad46221b2a187905e5266362b99d5e91c6ce24d165dab93e86433"],
+]
+
+INVALID_ADDRESS = [
+    "tc1qw508d6qejxtdg4y5r3zarvary0c5xw7kg3g4ty",
+    "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t5",
+    "BC13W508D6QEJXTDG4Y5R3ZARVARY0C5XW7KN40WF2",
+    "bc1rw5uspcuh",
+    "bc10w508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7kw5rljs90",
+    "BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P",
+    "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sL5k7",
+    "bc1zw508d6qejxtdg4y5r3zarvaryvqyzf3du",
+    "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv",
+    "bc1gmk9yu",
+
+]
+
+INVALID_ADDRESS_ENC = [
+    ("BC", 0, 20),
+    ("bc", 0, 21),
+    ("bc", 17, 32),
+    ("bc", 1, 1),
+    ("bc", 16, 41),
+]
+
+class TestSegwitAddress(unittest.TestCase):
+    """Unit test class for segwit addressess."""
+
+    def test_valid_checksum(self):
+        """Test checksum creation and validation."""
+        for test in VALID_CHECKSUM:
+            hrp, _ = btc.bech32_decode(test)
+            self.assertIsNotNone(hrp)
+            pos = test.rfind('1')
+            test = test[:pos+1] + chr(ord(test[pos + 1]) ^ 1) + test[pos+2:]
+            hrp, _ = btc.bech32_decode(test)
+            self.assertIsNone(hrp)
+
+    def test_invalid_checksum(self):
+        """Test validation of invalid checksums."""
+        for test in INVALID_CHECKSUM:
+            hrp, _ = btc.bech32_decode(test)
+            self.assertIsNone(hrp)
+
+    def test_valid_address(self):
+        """Test whether valid addresses decode to the correct output."""
+        for (address, hexscript) in VALID_ADDRESS:
+            hrp = "bc"
+            witver, witprog = btc.bech32addr_decode(hrp, address)
+            if witver is None:
+                hrp = "tb"
+                witver, witprog = btc.bech32addr_decode(hrp, address)
+            self.assertIsNotNone(witver)
+            scriptpubkey = segwit_scriptpubkey(witver, witprog)
+            self.assertEqual(scriptpubkey, binascii.unhexlify(hexscript))
+            addr = btc.bech32addr_encode(hrp, witver, witprog)
+            self.assertEqual(address.lower(), addr)
+
+    def test_invalid_address(self):
+        """Test whether invalid addresses fail to decode."""
+        for test in INVALID_ADDRESS:
+            witver, _ = btc.bech32addr_decode("bc", test)
+            self.assertIsNone(witver)
+            witver, _ = btc.bech32addr_decode("tb", test)
+            self.assertIsNone(witver)
+
+    def test_invalid_address_enc(self):
+        """Test whether address encoding fails on invalid input."""
+        for hrp, version, length in INVALID_ADDRESS_ENC:
+            code = btc.bech32addr_encode(hrp, version, [0] * length)
+            self.assertIsNone(code)
+
+if __name__ == "__main__":
+    unittest.main()

jmclient/jmclient/blockchaininterface.py

@@ -94,9 +94,14 @@ class BlockchainInterface(object):
             one_addr_imported = False
             for outs in txd['outs']:
                 addr = btc.script_to_address(outs['script'], vb)
-                if self.rpc('getaccount', [addr]) != '':
+                try:
-                    one_addr_imported = True
+                    if self.rpc('getaccount', [addr]) != '':
-                    break
+                        one_addr_imported = True
+                        break
+                except JsonRpcError as e:
+                    log.debug("Failed to getaccount for address: " + addr)
+                    log.debug("This is normal for bech32 addresses.")
+                    continue
             if not one_addr_imported:
                 self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False])
 

jmclient/jmclient/configure.py

@@ -267,6 +267,17 @@ def get_p2pk_vbyte():
 
 def validate_address(addr):
     try:
+        assert len(addr) > 2
+        if addr[:2].lower() in ['bc', 'tb']:
+            #Enforce testnet/mainnet per config
+            if get_network() == "testnet":
+                hrpreq = 'tb'
+            else:
+                hrpreq = 'bc'
+            if btc.bech32addr_decode(hrpreq, addr)[1]:
+                return True, 'address validated'
+            return False, 'Invalid bech32 address'
+        #Not bech32; assume b58 from here
         ver = btc.get_version_byte(addr)
     except AssertionError:
         return False, 'Checksum wrong. Typo in address?'

jmclient/jmclient/taker.py

@@ -655,13 +655,20 @@ class Taker(object):
         jlog.debug('\n' + tx)
         self.txid = btc.txhash(tx)
         jlog.info('txid = ' + self.txid)
+        #If we are sending to a bech32 address, in case of sweep, will
+        #need to use that bech32 for address import, which requires
+        #converting to script (Core does not allow import of bech32)
+        if self.my_cj_addr.lower()[:2] in ['bc', 'tb']:
+            notify_addr = btc.address_to_script(self.my_cj_addr)
+        else:
+            notify_addr = self.my_cj_addr
         #add the txnotify callbacks *before* pushing in case the
         #walletnotify is triggered before the notify callbacks are set up;
         #this does leave a dangling notify callback if the push fails, but
         #that doesn't cause problems.
         jm_single().bc_interface.add_tx_notify(self.latest_tx,
                     self.unconfirm_callback, self.confirm_callback,
-                    self.my_cj_addr, vb=get_p2sh_vbyte())
+                    notify_addr, vb=get_p2sh_vbyte())
         tx_broadcast = jm_single().config.get('POLICY', 'tx_broadcast')
         nick_to_use = None
         if tx_broadcast == 'self':

jmclient/test/test_valid_addresses.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 from jmclient.configure import validate_address, load_program_config
 from jmclient import jm_single
 import json
@@ -33,10 +34,44 @@ def test_b58_valid_addresses():
             else:
                 jm_single().config.set("BLOCKCHAIN", "network", "mainnet")
             #if using py.test -s ; sanity check to see what's actually being tested
-            print 'testing this address: ' + addr
+            print('testing this address: ', addr)
             res, message = validate_address(addr)
             assert res == True, "Incorrectly failed to validate address: " + addr + " with message: " + message
+    jm_single().config.set("BLOCKCHAIN", "network", "testnet")
 
+def test_valid_bech32_addresses():
+    valids = ["BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4",
+    "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7",
+    "bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx",
+    "BC1SW50QA3JX3S",
+    "bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj",
+    "tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy"]
+    for va in valids:
+        print("Testing this address: ", va)
+        if va.lower()[:2] == "bc":
+            jm_single().config.set("BLOCKCHAIN", "network", "mainnet")
+        else:
+            jm_single().config.set("BLOCKCHAIN", "network", "testnet")
+        res, message = validate_address(va)
+        assert res == True, "Incorrect failed to validate address: " + va + " with message: " + message
+    jm_single().config.set("BLOCKCHAIN", "network", "testnet")
+
+def test_invalid_bech32_addresses():
+    invalids = [
+    "tc1qw508d6qejxtdg4y5r3zarvary0c5xw7kg3g4ty",
+    "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t5",
+    "BC13W508D6QEJXTDG4Y5R3ZARVARY0C5XW7KN40WF2",
+    "bc1rw5uspcuh",
+    "bc10w508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7kw5rljs90",
+    "BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P",
+    "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sL5k7",
+    "bc1zw508d6qejxtdg4y5r3zarvaryvqyzf3du",
+    "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv",
+    "bc1gmk9yu"]
+    for iva in invalids:
+        print("Testing this address: ", iva)
+        res, message = validate_address(iva)
+        assert res == False, "Incorrectly validated address: " + iva
 
 @pytest.fixture(scope="module")
 def setup_addresses():

scripts/joinmarket-qt.py

@@ -893,7 +893,7 @@ class SpendTab(QWidget):
         self.tumbler_destaddrs = None
 
     def validateSettings(self):
-        valid, errmsg = validate_address(self.widgets[0][1].text())
+        valid, errmsg = validate_address(str(self.widgets[0][1].text()))
         if not valid:
             JMQtMessageBox(self, errmsg, mbtype='warn', title="Error")
             return False