diff --git a/base/commands.py b/base/commands.py new file mode 100644 index 0000000..7dfe709 --- /dev/null +++ b/base/commands.py @@ -0,0 +1,92 @@ +from twisted.protocols.amp import Integer, String, Unicode, Boolean, Command + +class DaemonNotReady(Exception): + pass + +class JMCommand(Command): + #a default response type + response = [('accepted', Boolean())] + +#commands from client to daemon + +class JMInit(JMCommand): + arguments = [('bcsource', String()), + ('network', String()), + ('irc_configs', String()), + ('minmakers', Integer()), + ('maker_timeout_sec', Integer())] + errors = {DaemonNotReady: 'daemon is not ready'} + +class JMStartMC(JMCommand): + arguments = [('nick', String())] + +class JMSetup(JMCommand): + arguments = [('role', String()), + ('n_counterparties', Integer())] + +class JMRequestOffers(JMCommand): + arguments = [] + +class JMFill(JMCommand): + arguments = [('amount', Integer()), + ('commitment', String()), + ('revelation', String()), + ('filled_offers', String())] + +class JMMakeTx(JMCommand): + arguments = [('nick_list', String()), + ('txhex', String())] + +class JMMsgSignature(JMCommand): + arguments = [('nick', String()), + ('cmd', String()), + ('msg_to_return', String()), + ('hostid', String())] + +class JMMsgSignatureVerify(JMCommand): + arguments = [('verif_result', Boolean()), + ('nick', String()), + ('fullmsg', String()), + ('hostid', String())] + +#commands from daemon to client + +class JMInitProto(JMCommand): + arguments = [('nick_hash_length', Integer()), + ('nick_max_encoded', Integer()), + ('joinmarket_nick_header', String()), + ('joinmarket_version', Integer())] + +class JMUp(JMCommand): + arguments = [] + +class JMSetupDone(JMCommand): + arguments = [] + +class JMOffers(JMCommand): + arguments = [('orderbook', String())] + +class JMFillResponse(JMCommand): + arguments = [('success', Boolean()), + ('ioauth_data', String())] + +class JMSigReceived(JMCommand): + arguments = [('nick', String()), + ('sig', String())] + +class JMRequestMsgSig(JMCommand): + arguments = [('nick', String()), + ('cmd', String()), + ('msg', String()), + ('msg_to_be_signed', String()), + ('hostid', String())] + +class JMRequestMsgSigVerify(JMCommand): + arguments = [('msg', String()), + ('fullmsg', String()), + ('sig', String()), + ('pubkey', String()), + ('nick', String()), + ('hashlen', Integer()), + ('max_encoded', Integer()), + ('hostid', String())] \ No newline at end of file diff --git a/base/support.py b/base/support.py new file mode 100644 index 0000000..2403782 --- /dev/null +++ b/base/support.py @@ -0,0 +1,69 @@ +from __future__ import absolute_import, print_function + +import sys + +import logging +import pprint +import random + +logFormatter = logging.Formatter( + "%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s") +log = logging.getLogger('joinmarket') +log.setLevel(logging.DEBUG) + +joinmarket_alert = [''] +core_alert = [''] +debug_silence = [False] + +#TODO pass this through from client, bitcoin paramater: +DUST_THRESHOLD = 2730 + +#consoleHandler = logging.StreamHandler(stream=sys.stdout) +class JoinMarketStreamHandler(logging.StreamHandler): + + def __init__(self, stream): + super(JoinMarketStreamHandler, self).__init__(stream) + + def emit(self, record): + if joinmarket_alert[0]: + print('JoinMarket Alert Message: ' + joinmarket_alert[0]) + if core_alert[0]: + print('Core Alert Message: ' + core_alert[0]) + if not debug_silence[0]: + super(JoinMarketStreamHandler, self).emit(record) + + +consoleHandler = JoinMarketStreamHandler(stream=sys.stdout) +consoleHandler.setFormatter(logFormatter) +log.addHandler(consoleHandler) + +log.debug('hello joinmarket') + + +def get_log(): + """ + provides joinmarket logging instance + :return: log instance + """ + return log + +def chunks(d, n): + return [d[x:x + n] for x in xrange(0, len(d), n)] + +def debug_dump_object(obj, skip_fields=None): + if skip_fields is None: + skip_fields = [] + log.debug('Class debug dump, name:' + obj.__class__.__name__) + for k, v in obj.__dict__.iteritems(): + if k in skip_fields: + continue + if k == 'password' or k == 'given_password': + continue + log.debug('key=' + k) + if isinstance(v, str): + log.debug('string: len:' + str(len(v))) + log.debug(v) + elif isinstance(v, dict) or isinstance(v, list): + log.debug(pprint.pformat(v)) + else: + log.debug(str(v)) diff --git a/bitcoin/__init__.py b/bitcoin/__init__.py new file mode 100644 index 0000000..829b48a --- /dev/null +++ b/bitcoin/__init__.py @@ -0,0 +1,6 @@ +import secp256k1 +from bitcoin.secp256k1_main import * +from bitcoin.secp256k1_transaction import * +from bitcoin.secp256k1_deterministic import * +from bitcoin.bci import * + diff --git a/bitcoin/bci.py b/bitcoin/bci.py new file mode 100644 index 0000000..05b1829 --- /dev/null +++ b/bitcoin/bci.py @@ -0,0 +1,73 @@ +#!/usr/bin/python +import json, re +import random +import sys +import time +import platform +from joinmarketclient.support import get_log +if platform.system() == "Windows": + import ssl + import urllib2 +else: + try: + from urllib.request import build_opener + except: + from urllib2 import build_opener + +log = get_log() + +# Makes a request to a given URL (first arg) and optional params (second arg) +def make_request(*args): + if platform.system() == "Windows": + sctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + sh = urllib2.HTTPSHandler(debuglevel=0, context=sctx) + opener = urllib2.build_opener(sh) + else: + opener = build_opener() + opener.addheaders = [('User-agent', + 'Mozilla/5.0' + str(random.randrange(1000000)))] + try: + return opener.open(*args).read().strip() + except Exception as e: + try: + p = e.read().strip() + except: + p = e + raise Exception(p) + +def make_request_blockr(*args): + counter = 0 + while True: + data = json.loads(make_request(*args)) + if data['status'] == 'error' and data['code'] == 429: + log.debug('Blockr service error: ' + data['message']) + time.sleep(min(60, 2**counter / 2.)) + counter += 1 + continue + return data + +# Pushes a transaction to the network using https://blockchain.info/pushtx +def bci_pushtx(tx): + if not re.match('^[0-9a-fA-F]*$', tx): + tx = tx.encode('hex') + return make_request('https://blockchain.info/pushtx', 'tx=' + tx) + +def blockr_pushtx(tx, network='btc'): + if network == 'testnet': + blockr_url = 'https://tbtc.blockr.io/api/v1/tx/push' + elif network == 'btc': + blockr_url = 'https://btc.blockr.io/api/v1/tx/push' + else: + raise Exception('Unsupported network {0} for blockr_pushtx'.format( + network)) + + if not re.match('^[0-9a-fA-F]*$', tx): + tx = tx.encode('hex') + return make_request(blockr_url, '{"hex":"%s"}' % tx) + + + + + + + diff --git a/bitcoin/secp256k1_deterministic.py b/bitcoin/secp256k1_deterministic.py new file mode 100644 index 0000000..b323973 --- /dev/null +++ b/bitcoin/secp256k1_deterministic.py @@ -0,0 +1,92 @@ +from bitcoin.secp256k1_main import * +import hmac +import hashlib +from binascii import hexlify + +# Below code ASSUMES binary inputs and compressed pubkeys +MAINNET_PRIVATE = b'\x04\x88\xAD\xE4' +MAINNET_PUBLIC = b'\x04\x88\xB2\x1E' +TESTNET_PRIVATE = b'\x04\x35\x83\x94' +TESTNET_PUBLIC = b'\x04\x35\x87\xCF' +PRIVATE = [MAINNET_PRIVATE, TESTNET_PRIVATE] +PUBLIC = [MAINNET_PUBLIC, TESTNET_PUBLIC] + +# BIP32 child key derivation + +def raw_bip32_ckd(rawtuple, i): + vbytes, depth, fingerprint, oldi, chaincode, key = rawtuple + i = int(i) + + if vbytes in PRIVATE: + priv = key + pub = privtopub(key, False) + else: + pub = key + + if i >= 2**31: + if vbytes in PUBLIC: + raise Exception("Can't do private derivation on public key!") + I = hmac.new(chaincode, b'\x00' + priv[:32] + encode(i, 256, 4), + hashlib.sha512).digest() + else: + I = hmac.new(chaincode, pub + encode(i, 256, 4), + hashlib.sha512).digest() + + if vbytes in PRIVATE: + newkey = add_privkeys(I[:32] + B'\x01', priv, False) + fingerprint = bin_hash160(privtopub(key, False))[:4] + if vbytes in PUBLIC: + newkey = add_pubkeys([privtopub(I[:32] + '\x01', False), key], False) + fingerprint = bin_hash160(key)[:4] + + return (vbytes, depth + 1, fingerprint, i, I[32:], newkey) + +def bip32_serialize(rawtuple): + vbytes, depth, fingerprint, i, chaincode, key = rawtuple + i = encode(i, 256, 4) + chaincode = encode(hash_to_int(chaincode), 256, 32) + keydata = b'\x00' + key[:-1] if vbytes in PRIVATE else key + bindata = vbytes + from_int_to_byte( + depth % 256) + fingerprint + i + chaincode + keydata + return changebase(bindata + bin_dbl_sha256(bindata)[:4], 256, 58) + +def bip32_deserialize(data): + dbin = changebase(data, 58, 256) + if bin_dbl_sha256(dbin[:-4])[:4] != dbin[-4:]: + raise Exception("Invalid checksum") + vbytes = dbin[0:4] + depth = from_byte_to_int(dbin[4]) + fingerprint = dbin[5:9] + i = decode(dbin[9:13], 256) + chaincode = dbin[13:45] + key = dbin[46:78] + b'\x01' if vbytes in PRIVATE else dbin[45:78] + return (vbytes, depth, fingerprint, i, chaincode, key) + +def raw_bip32_privtopub(rawtuple): + vbytes, depth, fingerprint, i, chaincode, key = rawtuple + newvbytes = MAINNET_PUBLIC if vbytes == MAINNET_PRIVATE else TESTNET_PUBLIC + return (newvbytes, depth, fingerprint, i, chaincode, privtopub(key, False)) + +def bip32_privtopub(data): + return bip32_serialize(raw_bip32_privtopub(bip32_deserialize(data))) + +def bip32_ckd(data, i): + return bip32_serialize(raw_bip32_ckd(bip32_deserialize(data), i)) + +def bip32_master_key(seed, vbytes=MAINNET_PRIVATE): + I = hmac.new( + from_string_to_bytes("Bitcoin seed"), seed, hashlib.sha512).digest() + return bip32_serialize((vbytes, 0, b'\x00' * 4, 0, I[32:], I[:32] + b'\x01' + )) + +def bip32_extract_key(data): + return safe_hexlify(bip32_deserialize(data)[-1]) + +def bip32_descend(*args): + if len(args) == 2: + key, path = args + else: + key, path = args[0], map(int, args[1:]) + for p in path: + key = bip32_ckd(key, p) + return bip32_extract_key(key) diff --git a/bitcoin/secp256k1_main.py b/bitcoin/secp256k1_main.py new file mode 100644 index 0000000..6df66bc --- /dev/null +++ b/bitcoin/secp256k1_main.py @@ -0,0 +1,393 @@ +#!/usr/bin/python +from __future__ import print_function +from .py2specials import * +from .py3specials import * +import binascii +import hashlib +import re +import sys +import os +import base64 +import time +import random +import hmac +import secp256k1 + +#Required only for PoDLE calculation: +N = 115792089237316195423570985008687907852837564279074904382605163141518161494337 +#Global context for secp256k1 operations (helps with performance) +ctx = secp256k1.lib.secp256k1_context_create(secp256k1.ALL_FLAGS) +#required for point addition +dummy_pub = secp256k1.PublicKey(ctx=ctx) + +#Standard prefix for Bitcoin message signing. +BITCOIN_MESSAGE_MAGIC = '\x18' + 'Bitcoin Signed Message:\n' + +"""A custom nonce function acting as a pass-through. +Only used for reusable donation pubkeys (stealth). +""" +from cffi import FFI + +ffi = FFI() +ffi.cdef('static int nonce_function_rand(unsigned char *nonce32,' + 'const unsigned char *msg32,const unsigned char *key32,' + 'const unsigned char *algo16,void *data,unsigned int attempt);') + +ffi.set_source("_noncefunc", +""" +static int nonce_function_rand(unsigned char *nonce32, +const unsigned char *msg32, +const unsigned char *key32, +const unsigned char *algo16, +void *data, +unsigned int attempt) +{ +memcpy(nonce32,data,32); +return 1; +} +""") + +ffi.compile() + +import _noncefunc +from _noncefunc import ffi + +def tweak_mul(point, scalar): + """Temporary hack because Windows binding had a bug in tweak_mul. + Can be removed when Windows binding is updated. + """ + return secp256k1._tweak_public(point, + secp256k1.lib.secp256k1_ec_pubkey_tweak_mul, + scalar) +"""PoDLE related primitives +""" +def getG(compressed=True): + """Returns the public key binary + representation of secp256k1 G + """ + priv = "\x00"*31 + "\x01" + G = secp256k1.PrivateKey(priv, ctx=ctx).pubkey.serialize(compressed) + return G + +podle_PublicKey_class = secp256k1.PublicKey +podle_PrivateKey_class = secp256k1.PrivateKey + +def podle_PublicKey(P): + """Returns a PublicKey object from a binary string + """ + return secp256k1.PublicKey(P, raw=True, ctx=ctx) + +def podle_PrivateKey(priv): + """Returns a PrivateKey object from a binary string + """ + return secp256k1.PrivateKey(priv, ctx=ctx) + + +def privkey_to_address(priv, from_hex=True, magicbyte=0): + return pubkey_to_address(privkey_to_pubkey(priv, from_hex), magicbyte) + +privtoaddr = privkey_to_address + +# Hashes +def bin_hash160(string): + intermed = hashlib.sha256(string).digest() + return hashlib.new('ripemd160', intermed).digest() + +def hash160(string): + return safe_hexlify(bin_hash160(string)) + +def bin_sha256(string): + binary_data = string if isinstance(string, bytes) else bytes(string, + 'utf-8') + return hashlib.sha256(binary_data).digest() + +def sha256(string): + return bytes_to_hex_string(bin_sha256(string)) + +def bin_dbl_sha256(s): + bytes_to_hash = from_string_to_bytes(s) + return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest() + +def dbl_sha256(string): + return safe_hexlify(bin_dbl_sha256(string)) + +def hash_to_int(x): + if len(x) in [40, 64]: + return decode(x, 16) + return decode(x, 256) + +def num_to_var_int(x): + x = int(x) + if x < 253: return from_int_to_byte(x) + elif x < 65536: return from_int_to_byte(253) + encode(x, 256, 2)[::-1] + elif x < 4294967296: return from_int_to_byte(254) + encode(x, 256, 4)[::-1] + else: return from_int_to_byte(255) + encode(x, 256, 8)[::-1] + +def message_sig_hash(message): + """Used for construction of signatures of + messages, intended to be compatible with Bitcoin Core. + """ + padded = BITCOIN_MESSAGE_MAGIC + num_to_var_int(len( + message)) + from_string_to_bytes(message) + return bin_dbl_sha256(padded) + +# Encodings +def b58check_to_bin(inp): + leadingzbytes = len(re.match('^1*', inp).group(0)) + data = b'\x00' * leadingzbytes + changebase(inp, 58, 256) + assert bin_dbl_sha256(data[:-4])[:4] == data[-4:] + return data[1:-4] + +def get_version_byte(inp): + leadingzbytes = len(re.match('^1*', inp).group(0)) + data = b'\x00' * leadingzbytes + changebase(inp, 58, 256) + assert bin_dbl_sha256(data[:-4])[:4] == data[-4:] + return ord(data[0]) + +def hex_to_b58check(inp, magicbyte=0): + return bin_to_b58check(binascii.unhexlify(inp), magicbyte) + +def b58check_to_hex(inp): + return safe_hexlify(b58check_to_bin(inp)) + +def pubkey_to_address(pubkey, magicbyte=0): + if len(pubkey) in [66, 130]: + return bin_to_b58check( + bin_hash160(binascii.unhexlify(pubkey)), magicbyte) + return bin_to_b58check(bin_hash160(pubkey), magicbyte) + +pubtoaddr = pubkey_to_address + +def wif_compressed_privkey(priv, vbyte=0): + """Convert privkey in hex compressed to WIF compressed + """ + if len(priv) != 66: + raise Exception("Wrong length of compressed private key") + if priv[-2:] != '01': + raise Exception("Private key has wrong compression byte") + return bin_to_b58check(binascii.unhexlify(priv), 128 + int(vbyte)) + + +def from_wif_privkey(wif_priv, compressed=True, vbyte=0): + """Convert WIF compressed privkey to hex compressed. + Caller specifies the network version byte (0 for mainnet, 0x6f + for testnet) that the key should correspond to; if there is + a mismatch an error is thrown. WIF encoding uses 128+ this number. + """ + bin_key = b58check_to_bin(wif_priv) + claimed_version_byte = get_version_byte(wif_priv) + if not 128+vbyte == claimed_version_byte: + raise Exception( + "WIF key version byte is wrong network (mainnet/testnet?)") + if compressed and not len(bin_key) == 33: + raise Exception("Compressed private key is not 33 bytes") + if compressed and not bin_key[-1] == '\x01': + raise Exception("Private key has incorrect compression byte") + return safe_hexlify(bin_key) + +def ecdsa_sign(msg, priv, usehex=True): + hashed_msg = message_sig_hash(msg) + if usehex: + #arguments to raw sign must be consistently hex or bin + hashed_msg = binascii.hexlify(hashed_msg) + sig = ecdsa_raw_sign(hashed_msg, priv, usehex, rawmsg=True) + #note those functions only handles binary, not hex + if usehex: + sig = binascii.unhexlify(sig) + return base64.b64encode(sig) + +def ecdsa_verify(msg, sig, pub, usehex=True): + hashed_msg = message_sig_hash(msg) + sig = base64.b64decode(sig) + if usehex: + #arguments to raw_verify must be consistently hex or bin + hashed_msg = binascii.hexlify(hashed_msg) + sig = binascii.hexlify(sig) + return ecdsa_raw_verify(hashed_msg, pub, sig, usehex, rawmsg=True) + +#Use secp256k1 to handle all EC and ECDSA operations. +#Data types: only hex and binary. +#Compressed and uncompressed private and public keys. +def hexbin(func): + '''To enable each function to 'speak' either hex or binary, + requires that the decorated function's final positional argument + is a boolean flag, True for hex and False for binary. + ''' + + def func_wrapper(*args, **kwargs): + if args[-1]: + newargs = [] + for arg in args[:-1]: + if isinstance(arg, (list, tuple)): + newargs += [[x.decode('hex') for x in arg]] + else: + newargs += [arg.decode('hex')] + newargs += [False] + returnval = func(*newargs, **kwargs) + if isinstance(returnval, bool): + return returnval + else: + return binascii.hexlify(returnval) + else: + return func(*args, **kwargs) + + return func_wrapper + +def read_privkey(priv): + if len(priv) == 33: + if priv[-1] == '\x01': + compressed = True + else: + raise Exception("Invalid private key") + elif len(priv) == 32: + compressed = False + else: + raise Exception("Invalid private key") + return (compressed, priv[:32]) + +@hexbin +def privkey_to_pubkey_inner(priv, usehex): + '''Take 32/33 byte raw private key as input. + If 32 bytes, return compressed (33 byte) raw public key. + If 33 bytes, read the final byte as compression flag, + and return compressed/uncompressed public key as appropriate.''' + compressed, priv = read_privkey(priv) + #secp256k1 checks for validity of key value. + newpriv = secp256k1.PrivateKey(privkey=priv, ctx=ctx) + return newpriv.pubkey.serialize(compressed=compressed) + +def privkey_to_pubkey(priv, usehex=True): + '''To avoid changing the interface from the legacy system, + allow an *optional* hex argument here (called differently from + maker/taker code to how it's called in bip32 code), then + pass to the standard hexbin decorator under the hood. + ''' + return privkey_to_pubkey_inner(priv, usehex) + +privtopub = privkey_to_pubkey + +@hexbin +def multiply(s, pub, usehex, rawpub=True, return_serialized=True): + '''Input binary compressed pubkey P(33 bytes) + and scalar s(32 bytes), return s*P. + The return value is a binary compressed public key, + or a PublicKey object if return_serialized is False. + Note that the called function does the type checking + of the scalar s. + ('raw' options passed in) + ''' + newpub = secp256k1.PublicKey(pub, raw=rawpub, ctx=ctx) + #see note to "tweak_mul" function in podle.py + res = secp256k1._tweak_public(newpub, + secp256k1.lib.secp256k1_ec_pubkey_tweak_mul, + s) + if not return_serialized: + return res + return res.serialize() + +@hexbin +def add_pubkeys(pubkeys, usehex): + '''Input a list of binary compressed pubkeys + and return their sum as a binary compressed pubkey.''' + r = secp256k1.PublicKey(ctx=ctx) #dummy holding object + pubkey_list = [secp256k1.PublicKey(x, + raw=True, + ctx=ctx).public_key for x in pubkeys] + r.combine(pubkey_list) + return r.serialize() + +@hexbin +def add_privkeys(priv1, priv2, usehex): + '''Add privkey 1 to privkey 2. + Input keys must be in binary either compressed or not. + Returned key will have the same compression state. + Error if compression state of both input keys is not the same.''' + y, z = [read_privkey(x) for x in [priv1, priv2]] + if y[0] != z[0]: + raise Exception("cannot add privkeys, mixed compression formats") + else: + compressed = y[0] + newpriv1, newpriv2 = (y[1], z[1]) + p1 = secp256k1.PrivateKey(newpriv1, raw=True, ctx=ctx) + res = p1.tweak_add(newpriv2) + if compressed: + res += '\x01' + return res + +@hexbin +def ecdsa_raw_sign(msg, + priv, + usehex, + rawpriv=True, + rawmsg=False, + usenonce=None): + '''Take the binary message msg and sign it with the private key + priv. + By default priv is just a 32 byte string, if rawpriv is false + it is assumed to be DER encoded. + If rawmsg is True, no sha256 hash is applied to msg before signing. + In this case, msg must be a precalculated hash (256 bit). + If rawmsg is False, the secp256k1 lib will hash the message as part + of the ECDSA-SHA256 signing algo. + If usenonce is not None, its value is passed to the secp256k1 library + sign() function as the ndata value, which is then used in conjunction + with a custom nonce generating function, such that the nonce used in the ECDSA + sign algorithm is exactly that value (ndata there, usenonce here). 32 bytes. + Return value: the calculated signature.''' + if rawmsg and len(msg) != 32: + raise Exception("Invalid hash input to ECDSA raw sign.") + if rawpriv: + compressed, p = read_privkey(priv) + newpriv = secp256k1.PrivateKey(p, raw=True, ctx=ctx) + else: + newpriv = secp256k1.PrivateKey(priv, raw=False, ctx=ctx) + if usenonce: + if len(usenonce) != 32: + raise ValueError("Invalid nonce passed to ecdsa_sign: " + str( + usenonce)) + nf = ffi.addressof(_noncefunc.lib, "nonce_function_rand") + ndata = ffi.new("char [32]", usenonce) + usenonce = (nf, ndata) + if usenonce: + sig = newpriv.ecdsa_sign(msg, raw=rawmsg, custom_nonce=usenonce) + else: + #partial fix for secp256k1-transient not including customnonce; + #partial because donations will crash on windows in the "if". + sig = newpriv.ecdsa_sign(msg, raw=rawmsg) + return newpriv.ecdsa_serialize(sig) + +@hexbin +def ecdsa_raw_verify(msg, pub, sig, usehex, rawmsg=False): + '''Take the binary message msg and binary signature sig, + and verify it against the pubkey pub. + If rawmsg is True, no sha256 hash is applied to msg before verifying. + In this case, msg must be a precalculated hash (256 bit). + If rawmsg is False, the secp256k1 lib will hash the message as part + of the ECDSA-SHA256 verification algo. + Return value: True if the signature is valid for this pubkey, False + otherwise. + Since the arguments may come from external messages their content is + not guaranteed, so return False on any parsing exception. + ''' + try: + if rawmsg: + assert len(msg) == 32 + newpub = secp256k1.PublicKey(pubkey=pub, raw=True, ctx=ctx) + sigobj = newpub.ecdsa_deserialize(sig) + retval = newpub.ecdsa_verify(msg, sigobj, raw=rawmsg) + except: + return False + return retval + +def estimate_tx_size(ins, outs, txtype='p2pkh'): + '''Estimate transaction size. + Assuming p2pkh: + out: 8+1+3+2+20=34, in: 1+32+4+1+1+~73+1+1+33=147, + ver:4,seq:4, +2 (len in,out) + total ~= 34*len_out + 147*len_in + 10 (sig sizes vary slightly) + ''' + if txtype == 'p2pkh': + return 10 + ins * 147 + 34 * outs + else: + raise NotImplementedError("Non p2pkh transaction size estimation not" + + "yet implemented") diff --git a/bitcoin/secp256k1_transaction.py b/bitcoin/secp256k1_transaction.py new file mode 100644 index 0000000..df8416f --- /dev/null +++ b/bitcoin/secp256k1_transaction.py @@ -0,0 +1,436 @@ +#!/usr/bin/python +import binascii, re, json, copy, sys +from bitcoin.secp256k1_main import * +from _functools import reduce +import os + +is_python2 = sys.version_info.major == 2 + +### Hex to bin converter and vice versa for objects +def json_is_base(obj, base): + if not is_python2 and isinstance(obj, bytes): + return False + + alpha = get_code_string(base) + if isinstance(obj, string_types): + for i in range(len(obj)): + if alpha.find(obj[i]) == -1: + return False + return True + elif isinstance(obj, int_types) or obj is None: + return True + elif isinstance(obj, list): + for i in range(len(obj)): + if not json_is_base(obj[i], base): + return False + return True + else: + for x in obj: + if not json_is_base(obj[x], base): + return False + return True + + +def json_changebase(obj, changer): + if isinstance(obj, string_or_bytes_types): + return changer(obj) + elif isinstance(obj, int_types) or obj is None: + return obj + elif isinstance(obj, list): + return [json_changebase(x, changer) for x in obj] + return dict((x, json_changebase(obj[x], changer)) for x in obj) + +# Transaction serialization and deserialization + + +def deserialize(tx): + if isinstance(tx, str) and re.match('^[0-9a-fA-F]*$', tx): + #tx = bytes(bytearray.fromhex(tx)) + return json_changebase( + deserialize(binascii.unhexlify(tx)), lambda x: safe_hexlify(x)) + # http://stackoverflow.com/questions/4851463/python-closure-write-to-variable-in-parent-scope + # Python's scoping rules are demented, requiring me to make pos an object + # so that it is call-by-reference + pos = [0] + + def read_as_int(bytez): + pos[0] += bytez + return decode(tx[pos[0] - bytez:pos[0]][::-1], 256) + + def read_var_int(): + pos[0] += 1 + + val = from_byte_to_int(tx[pos[0] - 1]) + if val < 253: + return val + return read_as_int(pow(2, val - 252)) + + def read_bytes(bytez): + pos[0] += bytez + return tx[pos[0] - bytez:pos[0]] + + def read_var_string(): + size = read_var_int() + return read_bytes(size) + + obj = {"ins": [], "outs": []} + obj["version"] = read_as_int(4) + ins = read_var_int() + for i in range(ins): + obj["ins"].append({ + "outpoint": { + "hash": read_bytes(32)[::-1], + "index": read_as_int(4) + }, + "script": read_var_string(), + "sequence": read_as_int(4) + }) + outs = read_var_int() + for i in range(outs): + obj["outs"].append({ + "value": read_as_int(8), + "script": read_var_string() + }) + obj["locktime"] = read_as_int(4) + return obj + + +def serialize(txobj): + #if isinstance(txobj, bytes): + # txobj = bytes_to_hex_string(txobj) + o = [] + if json_is_base(txobj, 16): + json_changedbase = json_changebase(txobj, + lambda x: binascii.unhexlify(x)) + hexlified = safe_hexlify(serialize(json_changedbase)) + return hexlified + o.append(encode(txobj["version"], 256, 4)[::-1]) + o.append(num_to_var_int(len(txobj["ins"]))) + for inp in txobj["ins"]: + o.append(inp["outpoint"]["hash"][::-1]) + o.append(encode(inp["outpoint"]["index"], 256, 4)[::-1]) + o.append(num_to_var_int(len(inp["script"])) + (inp["script"] if inp[ + "script"] or is_python2 else bytes())) + o.append(encode(inp["sequence"], 256, 4)[::-1]) + o.append(num_to_var_int(len(txobj["outs"]))) + for out in txobj["outs"]: + o.append(encode(out["value"], 256, 8)[::-1]) + o.append(num_to_var_int(len(out["script"])) + out["script"]) + o.append(encode(txobj["locktime"], 256, 4)[::-1]) + + return ''.join(o) if is_python2 else reduce(lambda x, y: x + y, o, bytes()) + +# Hashing transactions for signing + +SIGHASH_ALL = 1 +SIGHASH_NONE = 2 +SIGHASH_SINGLE = 3 +SIGHASH_ANYONECANPAY = 0x80 + +def signature_form(tx, i, script, hashcode=SIGHASH_ALL): + i, hashcode = int(i), int(hashcode) + if isinstance(tx, string_or_bytes_types): + return serialize(signature_form(deserialize(tx), i, script, hashcode)) + newtx = copy.deepcopy(tx) + for inp in newtx["ins"]: + inp["script"] = "" + newtx["ins"][i]["script"] = script + if hashcode & 0x1f == SIGHASH_NONE: + newtx["outs"] = [] + for j, inp in enumerate(newtx["ins"]): + if j != i: + inp["sequence"] = 0 + elif hashcode & 0x1f == SIGHASH_SINGLE: + if len(newtx["ins"]) > len(newtx["outs"]): + raise Exception( + "Transactions with sighash single should have len in <= len out") + newtx["outs"] = newtx["outs"][:i+1] + for out in newtx["outs"][:i]: + out['value'] = 2**64 - 1 + out['script'] = "" + for j, inp in enumerate(newtx["ins"]): + if j != i: + inp["sequence"] = 0 + if hashcode & SIGHASH_ANYONECANPAY: + newtx["ins"] = [newtx["ins"][i]] + else: + pass + return newtx + +def txhash(tx, hashcode=None): + if isinstance(tx, str) and re.match('^[0-9a-fA-F]*$', tx): + tx = changebase(tx, 16, 256) + if hashcode: + return dbl_sha256(from_string_to_bytes(tx) + encode( + int(hashcode), 256, 4)[::-1]) + else: + return safe_hexlify(bin_dbl_sha256(tx)[::-1]) + + +def bin_txhash(tx, hashcode=None): + return binascii.unhexlify(txhash(tx, hashcode)) + + +def ecdsa_tx_sign(tx, priv, hashcode=SIGHASH_ALL, usenonce=None): + sig = ecdsa_raw_sign( + txhash(tx, hashcode), + priv, + True, + rawmsg=True, + usenonce=usenonce) + return sig + encode(hashcode, 16, 2) + + +def ecdsa_tx_verify(tx, sig, pub, hashcode=SIGHASH_ALL): + return ecdsa_raw_verify( + txhash(tx, hashcode), + pub, + sig[:-2], + True, + rawmsg=True) + +# Scripts + + +def mk_pubkey_script(addr): + # Keep the auxiliary functions around for altcoins' sake + return '76a914' + b58check_to_hex(addr) + '88ac' + + +def mk_scripthash_script(addr): + return 'a914' + b58check_to_hex(addr) + '87' + +# Address representation to output script + + +def address_to_script(addr): + if addr[0] == '3' or addr[0] == '2': + return mk_scripthash_script(addr) + else: + return mk_pubkey_script(addr) + +# Output script to address representation + + +def script_to_address(script, vbyte=0): + if re.match('^[0-9a-fA-F]*$', script): + script = binascii.unhexlify(script) + if script[:3] == b'\x76\xa9\x14' and script[-2:] == b'\x88\xac' and len( + script) == 25: + return bin_to_b58check(script[3:-2], vbyte) # pubkey hash addresses + else: + if vbyte in [111, 196]: + # Testnet + scripthash_byte = 196 + else: + scripthash_byte = 5 + # BIP0016 scripthash addresses + return bin_to_b58check(script[2:-1], scripthash_byte) + + +def p2sh_scriptaddr(script, magicbyte=5): + if re.match('^[0-9a-fA-F]*$', script): + script = binascii.unhexlify(script) + return hex_to_b58check(hash160(script), magicbyte) + + +scriptaddr = p2sh_scriptaddr + + +def deserialize_script(script): + if isinstance(script, str) and re.match('^[0-9a-fA-F]*$', script): + return json_changebase( + deserialize_script(binascii.unhexlify(script)), + lambda x: safe_hexlify(x)) + out, pos = [], 0 + while pos < len(script): + code = from_byte_to_int(script[pos]) + if code == 0: + out.append(None) + pos += 1 + elif code <= 75: + out.append(script[pos + 1:pos + 1 + code]) + pos += 1 + code + elif code <= 78: + szsz = pow(2, code - 76) + sz = decode(script[pos + szsz:pos:-1], 256) + out.append(script[pos + 1 + szsz:pos + 1 + szsz + sz]) + pos += 1 + szsz + sz + elif code <= 96: + out.append(code - 80) + pos += 1 + else: + out.append(code) + pos += 1 + return out + + +def serialize_script_unit(unit): + if isinstance(unit, int): + if unit < 16: + return from_int_to_byte(unit + 80) + else: + return bytes([unit]) + elif unit is None: + return b'\x00' + else: + if len(unit) <= 75: + return from_int_to_byte(len(unit)) + unit + elif len(unit) < 256: + return from_int_to_byte(76) + from_int_to_byte(len(unit)) + unit + elif len(unit) < 65536: + return from_int_to_byte(77) + encode(len(unit), 256, 2)[::-1] + unit + else: + return from_int_to_byte(78) + encode(len(unit), 256, 4)[::-1] + unit + + +if is_python2: + + def serialize_script(script): + if json_is_base(script, 16): + return binascii.hexlify(serialize_script(json_changebase( + script, lambda x: binascii.unhexlify(x)))) + return ''.join(map(serialize_script_unit, script)) +else: + + def serialize_script(script): + if json_is_base(script, 16): + return safe_hexlify(serialize_script(json_changebase( + script, lambda x: binascii.unhexlify(x)))) + + result = bytes() + for b in map(serialize_script_unit, script): + result += b if isinstance(b, bytes) else bytes(b, 'utf-8') + return result + + +def mk_multisig_script(*args): # [pubs],k or pub1,pub2...pub[n],k + if isinstance(args[0], list): + pubs, k = args[0], int(args[1]) + else: + pubs = list(filter(lambda x: len(str(x)) >= 32, args)) + k = int(args[len(pubs)]) + return serialize_script([k] + pubs + [len(pubs)]) + 'ae' + +# Signing and verifying + + +def verify_tx_input(tx, i, script, sig, pub): + if re.match('^[0-9a-fA-F]*$', tx): + tx = binascii.unhexlify(tx) + if re.match('^[0-9a-fA-F]*$', script): + script = binascii.unhexlify(script) + if not re.match('^[0-9a-fA-F]*$', sig): + sig = safe_hexlify(sig) + if not re.match('^[0-9a-fA-F]*$', pub): + pub = safe_hexlify(pub) + hashcode = decode(sig[-2:], 16) + modtx = signature_form(tx, int(i), script, hashcode) + return ecdsa_tx_verify(modtx, sig, pub, hashcode) + + +def sign(tx, i, priv, hashcode=SIGHASH_ALL, usenonce=None): + i = int(i) + if (not is_python2 and isinstance(re, bytes)) or not re.match( + '^[0-9a-fA-F]*$', tx): + return binascii.unhexlify(sign(safe_hexlify(tx), i, priv)) + if len(priv) <= 33: + priv = safe_hexlify(priv) + pub = privkey_to_pubkey(priv, True) + address = pubkey_to_address(pub) + signing_tx = signature_form(tx, i, mk_pubkey_script(address), hashcode) + sig = ecdsa_tx_sign(signing_tx, priv, hashcode, usenonce=usenonce) + txobj = deserialize(tx) + txobj["ins"][i]["script"] = serialize_script([sig, pub]) + return serialize(txobj) + + +def signall(tx, priv): + # if priv is a dictionary, assume format is + # { 'txinhash:txinidx' : privkey } + if isinstance(priv, dict): + for e, i in enumerate(deserialize(tx)["ins"]): + k = priv["%s:%d" % (i["outpoint"]["hash"], i["outpoint"]["index"])] + tx = sign(tx, e, k) + else: + for i in range(len(deserialize(tx)["ins"])): + tx = sign(tx, i, priv) + return tx + + +def multisign(tx, i, script, pk, hashcode=SIGHASH_ALL): + if re.match('^[0-9a-fA-F]*$', tx): + tx = binascii.unhexlify(tx) + if re.match('^[0-9a-fA-F]*$', script): + script = binascii.unhexlify(script) + modtx = signature_form(tx, i, script, hashcode) + return ecdsa_tx_sign(modtx, pk, hashcode) + + +def apply_multisignatures(*args): + # tx,i,script,sigs OR tx,i,script,sig1,sig2...,sig[n] + tx, i, script = args[0], int(args[1]), args[2] + sigs = args[3] if isinstance(args[3], list) else list(args[3:]) + + if isinstance(script, str) and re.match('^[0-9a-fA-F]*$', script): + script = binascii.unhexlify(script) + sigs = [binascii.unhexlify(x) if x[:2] == '30' else x for x in sigs] + if isinstance(tx, str) and re.match('^[0-9a-fA-F]*$', tx): + return safe_hexlify(apply_multisignatures( + binascii.unhexlify(tx), i, script, sigs)) + + txobj = deserialize(tx) + txobj["ins"][i]["script"] = serialize_script([None] + sigs + [script]) + return serialize(txobj) + + +def is_inp(arg): + return len(arg) > 64 or "output" in arg or "outpoint" in arg + + +def mktx(*args): + # [in0, in1...],[out0, out1...] or in0, in1 ... out0 out1 ... + ins, outs = [], [] + for arg in args: + if isinstance(arg, list): + for a in arg: + (ins if is_inp(a) else outs).append(a) + else: + (ins if is_inp(arg) else outs).append(arg) + + txobj = {"locktime": 0, "version": 1, "ins": [], "outs": []} + for i in ins: + if isinstance(i, dict) and "outpoint" in i: + txobj["ins"].append(i) + else: + if isinstance(i, dict) and "output" in i: + i = i["output"] + txobj["ins"].append({ + "outpoint": {"hash": i[:64], + "index": int(i[65:])}, + "script": "", + "sequence": 4294967295 + }) + for o in outs: + if isinstance(o, string_or_bytes_types): + addr = o[:o.find(':')] + val = int(o[o.find(':') + 1:]) + o = {} + if re.match('^[0-9a-fA-F]*$', addr): + o["script"] = addr + else: + o["address"] = addr + o["value"] = val + + outobj = {} + if "address" in o: + outobj["script"] = address_to_script(o["address"]) + elif "script" in o: + outobj["script"] = o["script"] + else: + raise Exception("Could not find 'address' or 'script' in output.") + outobj["value"] = o["value"] + txobj["outs"].append(outobj) + + return serialize(txobj) + diff --git a/client/__init__.py b/client/__init__.py new file mode 100644 index 0000000..253e962 --- /dev/null +++ b/client/__init__.py @@ -0,0 +1,38 @@ +from __future__ import print_function + +import logging + +#Full joinmarket uses its own bitcoin module; +#other implementations (like wallet plugins) +#can optionally include their own, which must +#be implemented as an interface in btc.py +from btc import * + +from .support import get_log, calc_cj_fee, debug_dump_object, \ + choose_sweep_orders, choose_orders, \ + pick_order, cheapest_order_choose, weighted_order_choose, \ + rand_norm_array, rand_pow_array, rand_exp_array, joinmarket_alert, core_alert +from .jsonrpc import JsonRpcError, JsonRpcConnectionError, JsonRpc +from .old_mnemonic import mn_decode, mn_encode +from .slowaes import decryptData, encryptData +from .taker import Taker +from .wallet import AbstractWallet, BitcoinCoreInterface, Wallet, \ + BitcoinCoreWallet, estimate_tx_fee, ElectrumWrapWallet +from .configure import load_program_config, jm_single, get_p2pk_vbyte, \ + get_network, jm_single, get_network, validate_address, get_irc_mchannels, \ + check_utxo_blacklist +from .blockchaininterface import BlockrInterface, BlockchainInterface +from .client_protocol import JMTakerClientProtocolFactory, start_reactor +from .podle import set_commitment_file, get_commitment_file +from .commands import * +# Set default logging handler to avoid "No handler found" warnings. + +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logging.getLogger(__name__).addHandler(NullHandler()) + diff --git a/client/blockchaininterface.py b/client/blockchaininterface.py new file mode 100644 index 0000000..f5a14b6 --- /dev/null +++ b/client/blockchaininterface.py @@ -0,0 +1,1024 @@ +from __future__ import print_function + +import BaseHTTPServer +import abc +import ast +import json +import os +import pprint +import random +import re +import sys +import threading +import time +import urllib +import urllib2 +import traceback +from decimal import Decimal + +import btc + +# This can be removed once CliJsonRpc is gone. +import subprocess + +from joinmarketclient.jsonrpc import JsonRpcConnectionError, JsonRpcError +from joinmarketclient.configure import get_p2pk_vbyte, jm_single +from joinmarketclient.support import get_log, chunks + +log = get_log() + + +class CliJsonRpc(object): + """ + Fake JsonRpc class that uses the Bitcoin CLI executable. This is used + as temporary fall back before we switch completely (and exclusively) + to the real JSON-RPC interface. + """ + + def __init__(self, cli, testnet): + self.cli = cli + if testnet: + self.cli.append("-testnet") + + def call(self, method, params): + fullCall = [] + fullCall.extend(self.cli) + fullCall.append(method) + for p in params: + if isinstance(p, basestring): + fullCall.append(p) + else: + fullCall.append(json.dumps(p)) + + res = subprocess.check_output(fullCall) + + if res == '': + return None + + try: + return json.loads(res) + except ValueError: + return res.strip() + + +def is_index_ahead_of_cache(wallet, mix_depth, forchange): + if mix_depth >= len(wallet.index_cache): + return True + return wallet.index[mix_depth][forchange] >= wallet.index_cache[mix_depth][ + forchange] + + +class BlockchainInterface(object): + __metaclass__ = abc.ABCMeta + + def __init__(self): + pass + + def sync_wallet(self, wallet): + self.sync_addresses(wallet) + self.sync_unspent(wallet) + + @abc.abstractmethod + def sync_addresses(self, wallet): + """Finds which addresses have been used and sets + wallet.index appropriately""" + + @abc.abstractmethod + def sync_unspent(self, wallet): + """Finds the unspent transaction outputs belonging to this wallet, + sets wallet.unspent """ + + @abc.abstractmethod + def add_tx_notify(self, + txd, + unconfirmfun, + confirmfun, + notifyaddr, + timeoutfun=None): + """ + Invokes unconfirmfun and confirmfun when tx is seen on the network + If timeoutfun not None, called with boolean argument that tells + whether this is the timeout for unconfirmed or confirmed + timeout for uncontirmed = False + """ + pass + + @abc.abstractmethod + def pushtx(self, txhex): + """pushes tx to the network, returns False if failed""" + pass + + @abc.abstractmethod + def query_utxo_set(self, txouts, includeconf=False): + """ + takes a utxo or a list of utxos + returns None if they are spend or unconfirmed + otherwise returns value in satoshis, address and output script + optionally return the coin age in number of blocks + """ + # address and output script contain the same information btw + + @abc.abstractmethod + def estimate_fee_per_kb(self, N): + '''Use the blockchain interface to + get an estimate of the transaction fee per kb + required for inclusion in the next N blocks. + ''' + + +class ElectrumWalletInterface(BlockchainInterface): + """A pseudo-blockchain interface using the existing + Electrum server connection in an Electrum wallet. + Usage requires calling set_wallet with a valid Electrum + wallet instance. + """ + + def __init__(self, testnet=False): + super(ElectrumWalletInterface, self).__init__() + if testnet: + raise NotImplementedError( + "Electrum doesnt yet have a testnet interface") + self.last_sync_unspent = 0 + + def set_wallet(self, wallet): + self.wallet = wallet + + def sync_addresses(self, wallet): + log.debug("Dummy electrum interface, no sync address") + + def sync_unspent(self, wallet): + log.debug("Dummy electrum interface, no sync unspent") + + def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): + log.debug("Dummy electrum interface, no add tx notify") + + def pushtx(self, txhex, timeout=10): + #synchronous send + log.debug("About to push over electrum") + from electrum.transaction import Transaction + etx = Transaction(txhex) + etx.deserialize() + tx_hash = etx.hash() + try: + retval = self.wallet.network.synchronous_get( + ('blockchain.transaction.broadcast', [str(etx)]), timeout) + except: + log.debug("Failed electrum push") + return False + if retval != tx_hash: + log.debug("Pushtx over Electrum returned wrong value: " + str( + retval)) + return False + log.debug("Pushed via Electrum successfully, hash: " + tx_hash) + return True + + def query_utxo_set(self, txout, includeconf=False): + """Needed even for a simple dummy interface. + """ + self.current_height = self.wallet.network.blockchain.local_height + if not isinstance(txout, list): + txout = [txout] + utxos = [[t[:64], int(t[65:])] for t in txout] + result = [] + for ut in utxos: + address = self.wallet.network.synchronous_get(( + 'blockchain.utxo.get_address', ut)) + try: + utxo_info = self.wallet.network.synchronous_get(( + "blockchain.address.listunspent", [address])) + except Exception as e: + log.debug("Got exception calling listunspent: " + repr(e)) + raise + utxo = None + for u in utxo_info: + if u['tx_hash'] == ut[0] and u['tx_pos'] == ut[1]: + utxo = u + if utxo is None: + raise Exception("UTXO Not Found") + r = { + 'value': u['value'], + 'address': address, + 'script': btc.address_to_script(address) + } + if includeconf: + if int(u['height']) in [0, -1]: + #-1 means unconfirmed inputs + r['confirms'] = 0 + else: + #+1 because if current height = tx height, that's 1 conf + r['confirms'] = int(self.current_height) - int(u['height']) + 1 + result.append(r) + return result + + def estimate_fee_per_kb(self, N): + fee = self.wallet.network.synchronous_get(('blockchain.estimatefee', [N] + )) + log.debug("Got fee: " + str(fee)) + fee_per_kb_sat = int(float(fee) * 100000000) + return fee_per_kb_sat + + +class BlockrInterface(BlockchainInterface): + BLOCKR_MAX_ADDR_REQ_COUNT = 20 + + def __init__(self, testnet=False): + super(BlockrInterface, self).__init__() + + # see bci.py in bitcoin module + self.network = 'testnet' if testnet else 'btc' + self.blockr_domain = 'tbtc' if testnet else 'btc' + self.last_sync_unspent = 0 + + def sync_addresses(self, wallet): + log.debug('downloading wallet history') + # sets Wallet internal indexes to be at the next unused address + for mix_depth in range(wallet.max_mix_depth): + for forchange in [0, 1]: + unused_addr_count = 0 + last_used_addr = '' + while (unused_addr_count < wallet.gaplimit or + not is_index_ahead_of_cache(wallet, mix_depth, + forchange)): + addrs = [wallet.get_new_addr(mix_depth, forchange) + for _ in range(self.BLOCKR_MAX_ADDR_REQ_COUNT)] + + # TODO send a pull request to pybitcointools + # because this surely should be possible with a function from it + blockr_url = 'https://' + self.blockr_domain + blockr_url += '.blockr.io/api/v1/address/txs/' + + data = btc.make_request_blockr(blockr_url + ','.join( + addrs))['data'] + for dat in data: + if dat['nb_txs'] != 0: + last_used_addr = dat['address'] + unused_addr_count = 0 + else: + unused_addr_count += 1 + if last_used_addr == '': + wallet.index[mix_depth][forchange] = 0 + else: + next_avail_idx = max([wallet.addr_cache[last_used_addr][ + 2] + 1, wallet.index_cache[mix_depth][forchange]]) + wallet.index[mix_depth][forchange] = next_avail_idx + + def sync_unspent(self, wallet): + # finds utxos in the wallet + st = time.time() + # dont refresh unspent dict more often than 10 minutes + rate_limit_time = 10 * 60 + if st - self.last_sync_unspent < rate_limit_time: + log.debug( + 'blockr sync_unspent() happened too recently (%dsec), skipping' + % (st - self.last_sync_unspent)) + return + wallet.unspent = {} + + addrs = wallet.addr_cache.keys() + if len(addrs) == 0: + log.debug('no tx used') + return + i = 0 + while i < len(addrs): + inc = min(len(addrs) - i, self.BLOCKR_MAX_ADDR_REQ_COUNT) + req = addrs[i:i + inc] + i += inc + + # TODO send a pull request to pybitcointools + # unspent() doesnt tell you which address, you get a bunch of utxos + # but dont know which privkey to sign with + + blockr_url = 'https://' + self.blockr_domain + \ + '.blockr.io/api/v1/address/unspent/' + data = btc.make_request_blockr(blockr_url + ','.join(req))['data'] + if 'unspent' in data: + data = [data] + for dat in data: + for u in dat['unspent']: + wallet.unspent[u['tx'] + ':' + str(u['n'])] = { + 'address': dat['address'], + 'value': int(u['amount'].replace('.', '')) + } + for u in wallet.spent_utxos: + wallet.unspent.pop(u, None) + + self.last_sync_unspent = time.time() + log.debug('blockr sync_unspent took ' + str((self.last_sync_unspent - st + )) + 'sec') + + def add_tx_notify(self, + txd, + unconfirmfun, + confirmfun, + notifyaddr, + timeoutfun=None): + unconfirm_timeout = jm_single().config.getint('TIMEOUT', + 'unconfirm_timeout_sec') + unconfirm_poll_period = 5 + confirm_timeout = jm_single().config.getfloat( + 'TIMEOUT', 'confirm_timeout_hours') * 60 * 60 + confirm_poll_period = 5 * 60 + + class NotifyThread(threading.Thread): + + def __init__(self, blockr_domain, txd, unconfirmfun, confirmfun, + timeoutfun): + threading.Thread.__init__(self, name='BlockrNotifyThread') + self.daemon = True + self.blockr_domain = blockr_domain + self.unconfirmfun = unconfirmfun + self.confirmfun = confirmfun + self.timeoutfun = timeoutfun + self.tx_output_set = set([(sv['script'], sv['value']) + for sv in txd['outs']]) + self.output_addresses = [ + btc.script_to_address(scrval[0], get_p2pk_vbyte()) + for scrval in self.tx_output_set + ] + log.debug('txoutset=' + pprint.pformat(self.tx_output_set)) + log.debug('outaddrs=' + ','.join(self.output_addresses)) + + def run(self): + st = int(time.time()) + unconfirmed_txid = None + unconfirmed_txhex = None + while not unconfirmed_txid: + time.sleep(unconfirm_poll_period) + if int(time.time()) - st > unconfirm_timeout: + log.debug('checking for unconfirmed tx timed out') + if self.timeoutfun: + self.timeoutfun(False) + return + blockr_url = 'https://' + self.blockr_domain + blockr_url += '.blockr.io/api/v1/address/unspent/' + random.shuffle(self.output_addresses + ) # seriously weird bug with blockr.io + data = btc.make_request_blockr(blockr_url + ','.join( + self.output_addresses) + '?unconfirmed=1')['data'] + + shared_txid = None + for unspent_list in data: + txs = set([str(txdata['tx']) + for txdata in unspent_list['unspent']]) + if not shared_txid: + shared_txid = txs + else: + shared_txid = shared_txid.intersection(txs) + log.debug('sharedtxid = ' + str(shared_txid)) + if len(shared_txid) == 0: + continue + time.sleep( + 2 + ) # here for some race condition bullshit with blockr.io + blockr_url = 'https://' + self.blockr_domain + blockr_url += '.blockr.io/api/v1/tx/raw/' + data = btc.make_request_blockr(blockr_url + ','.join( + shared_txid))['data'] + if not isinstance(data, list): + data = [data] + for txinfo in data: + txhex = str(txinfo['tx']['hex']) + outs = set([(sv['script'], sv['value']) + for sv in btc.deserialize(txhex)['outs']]) + log.debug('unconfirm query outs = ' + str(outs)) + if outs == self.tx_output_set: + unconfirmed_txid = txinfo['tx']['txid'] + unconfirmed_txhex = str(txinfo['tx']['hex']) + break + + self.unconfirmfun( + btc.deserialize(unconfirmed_txhex), unconfirmed_txid) + + st = int(time.time()) + confirmed_txid = None + confirmed_txhex = None + while not confirmed_txid: + time.sleep(confirm_poll_period) + if int(time.time()) - st > confirm_timeout: + log.debug('checking for confirmed tx timed out') + if self.timeoutfun: + self.timeoutfun(True) + return + blockr_url = 'https://' + self.blockr_domain + blockr_url += '.blockr.io/api/v1/address/txs/' + data = btc.make_request_blockr(blockr_url + ','.join( + self.output_addresses))['data'] + shared_txid = None + for addrtxs in data: + txs = set(str(txdata['tx']) + for txdata in addrtxs['txs']) + if not shared_txid: + shared_txid = txs + else: + shared_txid = shared_txid.intersection(txs) + log.debug('sharedtxid = ' + str(shared_txid)) + if len(shared_txid) == 0: + continue + blockr_url = 'https://' + self.blockr_domain + blockr_url += '.blockr.io/api/v1/tx/raw/' + data = btc.make_request_blockr(blockr_url + ','.join( + shared_txid))['data'] + if not isinstance(data, list): + data = [data] + for txinfo in data: + txhex = str(txinfo['tx']['hex']) + outs = set([(sv['script'], sv['value']) + for sv in btc.deserialize(txhex)['outs']]) + log.debug('confirm query outs = ' + str(outs)) + if outs == self.tx_output_set: + confirmed_txid = txinfo['tx']['txid'] + confirmed_txhex = str(txinfo['tx']['hex']) + break + self.confirmfun( + btc.deserialize(confirmed_txhex), confirmed_txid, 1) + + NotifyThread(self.blockr_domain, txd, unconfirmfun, confirmfun, + timeoutfun).start() + + def pushtx(self, txhex): + try: + json_str = btc.blockr_pushtx(txhex, self.network) + data = json.loads(json_str) + if data['status'] != 'success': + log.debug(data) + return False + except Exception: + log.debug('failed blockr.io pushtx') + log.debug(traceback.format_exc()) + return False + return True + + def query_utxo_set(self, txout, includeconf=False): + if not isinstance(txout, list): + txout = [txout] + txids = [h[:64] for h in txout] + txids = list(set(txids)) # remove duplicates + # self.BLOCKR_MAX_ADDR_REQ_COUNT = 2 + if len(txids) > self.BLOCKR_MAX_ADDR_REQ_COUNT: + txids = chunks(txids, self.BLOCKR_MAX_ADDR_REQ_COUNT) + else: + txids = [txids] + data = [] + for ids in txids: + blockr_url = 'https://' + self.blockr_domain + '.blockr.io/api/v1/tx/info/' + data = btc.make_request_blockr(blockr_url + ','.join(ids))['data'] + if not isinstance(blockr_data, list): + blockr_data = [blockr_data] + data += blockr_data + result = [] + for txo in txout: + txdata = [d for d in data if d['tx'] == txo[:64]][0] + vout = [v for v in txdata['vouts'] if v['n'] == int(txo[65:])][0] + if "is_spent" in vout and vout['is_spent'] == 1: + result.append(None) + else: + result_dict = {'value': int(Decimal(vout['amount']) * + Decimal('1e8')), + 'address': vout['address'], + 'script': vout['extras']['script']} + if includeconf: + result_dict['confirms'] = int(txdata['confirmations']) + result.append(result_dict) + return result + + def estimate_fee_per_kb(self, N): + bcypher_fee_estimate_url = 'https://api.blockcypher.com/v1/btc/main' + bcypher_data = json.loads(btc.make_request(bcypher_fee_estimate_url)) + log.debug("Got blockcypher result: " + pprint.pformat(bcypher_data)) + if N <= 2: + fee_per_kb = bcypher_data["high_fee_per_kb"] + elif N <= 4: + fee_per_kb = bcypher_data["medium_fee_per_kb"] + else: + fee_per_kb = bcypher_data["low_fee_per_kb"] + + return fee_per_kb + + +def bitcoincore_timeout_callback(uc_called, txout_set, txnotify_fun_list, + timeoutfun): + log.debug('bitcoin core timeout callback uc_called = %s' % ('true' + if uc_called + else 'false')) + txnotify_tuple = None + for tnf in txnotify_fun_list: + if tnf[0] == txout_set and uc_called == tnf[-1]: + txnotify_tuple = tnf + break + if txnotify_tuple == None: + log.debug('stale timeout, returning') + return + txnotify_fun_list.remove(txnotify_tuple) + log.debug('timeoutfun txout_set=\n' + pprint.pformat(txout_set)) + timeoutfun(uc_called) + + +class NotifyRequestHeader(BaseHTTPServer.BaseHTTPRequestHandler): + + def __init__(self, request, client_address, base_server): + self.btcinterface = base_server.btcinterface + self.base_server = base_server + BaseHTTPServer.BaseHTTPRequestHandler.__init__( + self, request, client_address, base_server) + + def do_HEAD(self): + pages = ('/walletnotify?', '/alertnotify?') + + if self.path.startswith('/walletnotify?'): + txid = self.path[len(pages[0]):] + if not re.match('^[0-9a-fA-F]*$', txid): + log.debug('not a txid') + return + try: + tx = self.btcinterface.rpc('getrawtransaction', [txid]) + except (JsonRpcError, JsonRpcConnectionError) as e: + log.debug('transaction not found, probably a conflict') + return + if not re.match('^[0-9a-fA-F]*$', tx): + log.debug('not a txhex') + return + txd = btc.deserialize(tx) + tx_output_set = set([(sv['script'], sv['value']) for sv in txd[ + 'outs']]) + + txnotify_tuple = None + unconfirmfun, confirmfun, timeoutfun, uc_called = (None, None, None, + None) + for tnf in self.btcinterface.txnotify_fun: + tx_out = tnf[0] + if tx_out == tx_output_set: + txnotify_tuple = tnf + tx_out, unconfirmfun, confirmfun, timeoutfun, uc_called = tnf + break + if unconfirmfun is None: + log.debug('txid=' + txid + ' not being listened for') + else: + # on rare occasions people spend their output without waiting + # for a confirm + txdata = None + for n in range(len(txd['outs'])): + txdata = self.btcinterface.rpc('gettxout', [txid, n, True]) + if txdata is not None: + break + assert txdata is not None + if txdata['confirmations'] == 0: + unconfirmfun(txd, txid) + # TODO pass the total transfered amount value here somehow + # wallet_name = self.get_wallet_name() + # amount = + # bitcoin-cli move wallet_name "" amount + self.btcinterface.txnotify_fun.remove(txnotify_tuple) + self.btcinterface.txnotify_fun.append(txnotify_tuple[:-1] + + (True,)) + log.debug('ran unconfirmfun') + if timeoutfun: + threading.Timer(jm_single().config.getfloat( + 'TIMEOUT', 'confirm_timeout_hours') * 60 * 60, + bitcoincore_timeout_callback, + args=(True, tx_output_set, + self.btcinterface.txnotify_fun, + timeoutfun)).start() + else: + if not uc_called: + unconfirmfun(txd, txid) + log.debug('saw confirmed tx before unconfirmed, ' + + 'running unconfirmfun first') + confirmfun(txd, txid, txdata['confirmations']) + self.btcinterface.txnotify_fun.remove(txnotify_tuple) + log.debug('ran confirmfun') + + elif self.path.startswith('/alertnotify?'): + jm_single().core_alert[0] = urllib.unquote(self.path[len(pages[ + 1]):]) + log.debug('Got an alert!\nMessage=' + jm_single().core_alert[0]) + + else: + log.debug( + 'ERROR: This is not a handled URL path. You may want to check your notify URL for typos.') + + request = urllib2.Request('http://localhost:' + str( + self.base_server.server_address[1] + 1) + self.path) + request.get_method = lambda: 'HEAD' + try: + urllib2.urlopen(request) + except urllib2.URLError: + pass + self.send_response(200) + # self.send_header('Connection', 'close') + self.end_headers() + + +class BitcoinCoreNotifyThread(threading.Thread): + + def __init__(self, btcinterface): + threading.Thread.__init__(self, name='CoreNotifyThread') + self.daemon = True + self.btcinterface = btcinterface + + def run(self): + notify_host = 'localhost' + notify_port = 62602 # defaults + config = jm_single().config + if 'notify_host' in config.options("BLOCKCHAIN"): + notify_host = config.get("BLOCKCHAIN", "notify_host").strip() + if 'notify_port' in config.options("BLOCKCHAIN"): + notify_port = int(config.get("BLOCKCHAIN", "notify_port")) + for inc in range(10): + hostport = (notify_host, notify_port + inc) + try: + httpd = BaseHTTPServer.HTTPServer(hostport, NotifyRequestHeader) + except Exception: + continue + httpd.btcinterface = self.btcinterface + log.debug('started bitcoin core notify listening thread, host=' + + str(notify_host) + ' port=' + str(hostport[1])) + httpd.serve_forever() + log.debug('failed to bind for bitcoin core notify listening') + +# must run bitcoind with -server +# -walletnotify="curl -sI --connect-timeout 1 http://localhost:62602/walletnotify?%s" +# and make sure curl is installed (git uses it, odds are you've already got it) + + +class BitcoinCoreInterface(BlockchainInterface): + + def __init__(self, jsonRpc, network): + super(BitcoinCoreInterface, self).__init__() + self.jsonRpc = jsonRpc + + blockchainInfo = self.jsonRpc.call("getblockchaininfo", []) + actualNet = blockchainInfo['chain'] + + netmap = {'main': 'mainnet', 'test': 'testnet', 'regtest': 'regtest'} + if netmap[actualNet] != network: + raise Exception('wrong network configured') + + self.notifythread = None + self.txnotify_fun = [] + self.wallet_synced = False + + @staticmethod + def get_wallet_name(wallet): + return 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] + + def rpc(self, method, args): + if method not in ['importaddress', 'walletpassphrase']: + log.debug('rpc: ' + method + " " + str(args)) + res = self.jsonRpc.call(method, args) + if isinstance(res, unicode): + res = str(res) + return res + + def add_watchonly_addresses(self, addr_list, wallet_name): + log.debug('importing ' + str(len(addr_list)) + + ' addresses into account ' + wallet_name) + for addr in addr_list: + self.rpc('importaddress', [addr, wallet_name, False]) + if jm_single().config.get("BLOCKCHAIN", + "blockchain_source") != 'regtest': + print('restart Bitcoin Core with -rescan if you\'re ' + 'recovering an existing wallet from backup seed') + print(' otherwise just restart this joinmarket script') + sys.exit(0) + + def sync_addresses(self, wallet): + from joinmarketclient.wallet import BitcoinCoreWallet + + if isinstance(wallet, BitcoinCoreWallet): + return + log.debug('requesting wallet history') + wallet_name = self.get_wallet_name(wallet) + #TODO It is worth considering making this user configurable: + addr_req_count = 20 + wallet_addr_list = [] + for mix_depth in range(wallet.max_mix_depth): + for forchange in [0, 1]: + #If we have an index-cache available, we can use it + #to decide how much to import (note that this list + #*always* starts from index 0 on each branch). + #In cases where the Bitcoin Core instance is fresh, + #this will allow the entire import+rescan to occur + #in 2 steps only. + if wallet.index_cache != [[0, 0]] * wallet.max_mix_depth: + #Need to request N*addr_req_count where N is least s.t. + #N*addr_req_count > index_cache val. This is so that the batching + #process in the main loop *always* has already imported enough + #addresses to complete. + req_count = int(wallet.index_cache[mix_depth][forchange] / + addr_req_count) + 1 + req_count *= addr_req_count + else: + #If we have *nothing* - no index_cache, and no info + #in Core wallet (imports), we revert to a batching mode + #with a default size. + #In this scenario it could require several restarts *and* + #rescans; perhaps user should set addr_req_count high + #(see above TODO) + req_count = addr_req_count + wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange) + for _ in range(req_count)] + #Indices are reset here so that the next algorithm step starts + #from the beginning of each branch + wallet.index[mix_depth][forchange] = 0 + # makes more sense to add these in an account called "joinmarket-imported" but its much + # simpler to add to the same account here + for privkey_list in wallet.imported_privkeys.values(): + for privkey in privkey_list: + imported_addr = btc.privtoaddr(privkey, + magicbyte=get_p2pk_vbyte()) + wallet_addr_list.append(imported_addr) + imported_addr_list = self.rpc('getaddressesbyaccount', [wallet_name]) + if not set(wallet_addr_list).issubset(set(imported_addr_list)): + self.add_watchonly_addresses(wallet_addr_list, wallet_name) + return + + buf = self.rpc('listtransactions', [wallet_name, 1000, 0, True]) + txs = buf + # If the buffer's full, check for more, until it ain't + while len(buf) == 1000: + buf = self.rpc('listtransactions', [wallet_name, 1000, len(txs), + True]) + txs += buf + # TODO check whether used_addr_list can be a set, may be faster (if + # its a hashset) and allows using issubset() here and setdiff() for + # finding which addresses need importing + + # TODO also check the fastest way to build up python lists, i suspect + # using += is slow + used_addr_list = [tx['address'] + for tx in txs if tx['category'] == 'receive'] + too_few_addr_mix_change = [] + for mix_depth in range(wallet.max_mix_depth): + for forchange in [0, 1]: + unused_addr_count = 0 + last_used_addr = '' + breakloop = False + while not breakloop: + if unused_addr_count >= wallet.gaplimit and \ + is_index_ahead_of_cache(wallet, mix_depth, + forchange): + break + mix_change_addrs = [ + wallet.get_new_addr(mix_depth, forchange) + for _ in range(addr_req_count) + ] + for mc_addr in mix_change_addrs: + if mc_addr not in imported_addr_list: + too_few_addr_mix_change.append((mix_depth, forchange + )) + breakloop = True + break + if mc_addr in used_addr_list: + last_used_addr = mc_addr + unused_addr_count = 0 + else: + unused_addr_count += 1 +#index setting here depends on whether we broke out of the loop +#early; if we did, it means we need to prepare the index +#at the level of the last used address or zero so as to not +#miss any imports in add_watchonly_addresses. +#If we didn't, we need to respect the index_cache to avoid +#potential address reuse. + if breakloop: + if last_used_addr == '': + wallet.index[mix_depth][forchange] = 0 + else: + wallet.index[mix_depth][forchange] = \ + wallet.addr_cache[last_used_addr][2] + 1 + else: + if last_used_addr == '': + next_avail_idx = max([wallet.index_cache[mix_depth][ + forchange], 0]) + else: + next_avail_idx = max([wallet.addr_cache[last_used_addr][ + 2] + 1, wallet.index_cache[mix_depth][forchange]]) + wallet.index[mix_depth][forchange] = next_avail_idx + + wallet_addr_list = [] + if len(too_few_addr_mix_change) > 0: + indices = [wallet.index[mc[0]][mc[1]] + for mc in too_few_addr_mix_change] + log.debug('too few addresses in ' + str(too_few_addr_mix_change) + + ' at ' + str(indices)) + for mix_depth, forchange in too_few_addr_mix_change: + wallet_addr_list += [ + wallet.get_new_addr(mix_depth, forchange) + for _ in range(addr_req_count * 3) + ] + + self.add_watchonly_addresses(wallet_addr_list, wallet_name) + return + + self.wallet_synced = True + + def sync_unspent(self, wallet): + from joinmarketclient.wallet import BitcoinCoreWallet + + if isinstance(wallet, BitcoinCoreWallet): + return + st = time.time() + wallet_name = self.get_wallet_name(wallet) + wallet.unspent = {} + + listunspent_args = [] + if 'listunspent_args' in jm_single().config.options('POLICY'): + listunspent_args = ast.literal_eval(jm_single().config.get( + 'POLICY', 'listunspent_args')) + + unspent_list = self.rpc('listunspent', listunspent_args) + for u in unspent_list: + if 'account' not in u: + continue + if u['account'] != wallet_name: + continue + if u['address'] not in wallet.addr_cache: + continue + wallet.unspent[u['txid'] + ':' + str(u['vout'])] = { + 'address': u['address'], + 'value': int(Decimal(str(u['amount'])) * Decimal('1e8')) + } + et = time.time() + log.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec') + + def add_tx_notify(self, + txd, + unconfirmfun, + confirmfun, + notifyaddr, + timeoutfun=None): + if not self.notifythread: + self.notifythread = BitcoinCoreNotifyThread(self) + self.notifythread.start() + one_addr_imported = False + for outs in txd['outs']: + addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) + if self.rpc('getaccount', [addr]) != '': + one_addr_imported = True + break + if not one_addr_imported: + self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False]) + tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) + self.txnotify_fun.append((tx_output_set, unconfirmfun, confirmfun, + timeoutfun, False)) + + #create unconfirm timeout here, create confirm timeout in the other thread + if timeoutfun: + threading.Timer(jm_single().config.getint('TIMEOUT', + 'unconfirm_timeout_sec'), + bitcoincore_timeout_callback, + args=(False, tx_output_set, self.txnotify_fun, + timeoutfun)).start() + + def pushtx(self, txhex): + try: + txid = self.rpc('sendrawtransaction', [txhex]) + except JsonRpcConnectionError as e: + log.debug('error pushing = ' + repr(e)) + return False + except JsonRpcError as e: + log.debug('error pushing = ' + str(e.code) + " " + str(e.message)) + return False + return True + + def query_utxo_set(self, txout, includeconf=False): + if not isinstance(txout, list): + txout = [txout] + result = [] + for txo in txout: + ret = self.rpc('gettxout', [txo[:64], int(txo[65:]), False]) + if ret is None: + result.append(None) + else: + result_dict = {'value': int(Decimal(str(ret['value'])) * + Decimal('1e8')), + 'address': ret['scriptPubKey']['addresses'][0], + 'script': ret['scriptPubKey']['hex']} + if includeconf: + result_dict['confirms'] = int(ret['confirmations']) + result.append(result_dict) + return result + + def estimate_fee_per_kb(self, N): + estimate = Decimal(1e8) * Decimal(self.rpc('estimatefee', [N])) + if estimate < 0: + #This occurs when Core has insufficient data to estimate. + #TODO anything better than a hardcoded default? + return 30000 + else: + return estimate + + +# class for regtest chain access +# running on local daemon. Only +# to be instantiated after network is up +# with > 100 blocks. +class RegtestBitcoinCoreInterface(BitcoinCoreInterface): + + def __init__(self, jsonRpc): + super(RegtestBitcoinCoreInterface, self).__init__(jsonRpc, 'regtest') + self.pushtx_failure_prob = 0 + self.tick_forward_chain_interval = 2 + self.absurd_fees = False + + def estimate_fee_per_kb(self, N): + if not self.absurd_fees: + return super(RegtestBitcoinCoreInterface, + self).estimate_fee_per_kb(N) + else: + return jm_single().config.getint("POLICY", + "absurd_fee_per_kb") + 100 + + def pushtx(self, txhex): + if self.pushtx_failure_prob != 0 and random.random() <\ + self.pushtx_failure_prob: + log.debug('randomly not broadcasting %0.1f%% of the time' % + (self.pushtx_failure_prob * 100)) + return True + + ret = super(RegtestBitcoinCoreInterface, self).pushtx(txhex) + + class TickChainThread(threading.Thread): + + def __init__(self, bcinterface): + threading.Thread.__init__(self, name='TickChainThread') + self.bcinterface = bcinterface + + def run(self): + if self.bcinterface.tick_forward_chain_interval < 0: + log.debug('not ticking forward chain') + return + time.sleep(self.bcinterface.tick_forward_chain_interval) + self.bcinterface.tick_forward_chain(1) + + TickChainThread(self).start() + return ret + + def tick_forward_chain(self, n): + """ + Special method for regtest only; + instruct to mine n blocks. + """ + try: + self.rpc('generate', [n]) + except JsonRpcConnectionError: + #can happen if the blockchain is shut down + #automatically at the end of tests; this shouldn't + #trigger an error + log.debug( + "Failed to generate blocks, looks like the bitcoin daemon \ + has been shut down. Ignoring.") + pass + + def grab_coins(self, receiving_addr, amt=50): + """ + NOTE! amt is passed in Coins, not Satoshis! + Special method for regtest only: + take coins from bitcoind's own wallet + and put them in the receiving addr. + Return the txid. + """ + if amt > 500: + raise Exception("too greedy") + """ + if amt > self.current_balance: + #mine enough to get to the reqd amt + reqd = int(amt - self.current_balance) + reqd_blocks = int(reqd/50) +1 + if self.rpc('setgenerate', [True, reqd_blocks]): + raise Exception("Something went wrong") + """ + # now we do a custom create transaction and push to the receiver + txid = self.rpc('sendtoaddress', [receiving_addr, amt]) + if not txid: + raise Exception("Failed to broadcast transaction") + # confirm + self.tick_forward_chain(1) + return txid + + def get_received_by_addr(self, addresses, query_params): + # NB This will NOT return coinbase coins (but wont matter in our use + # case). allow importaddress to fail in case the address is already + # in the wallet + res = [] + for address in addresses: + self.rpc('importaddress', [address, 'watchonly']) + res.append({'address': address, + 'balance': int(round(Decimal(1e8) * Decimal(self.rpc( + 'getreceivedbyaddress', [address]))))}) + return {'data': res} + +# todo: won't run anyways +# def main(): +# #TODO some useful quick testing here, so people know if they've set it up right +# myBCI = RegtestBitcoinCoreInterface() +# #myBCI.send_tx('stuff') +# print myBCI.get_utxos_from_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) +# print myBCI.get_balance_at_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) +# txid = myBCI.grab_coins('mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd', 23) +# print txid +# print myBCI.get_balance_at_addr(['mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd']) +# print myBCI.get_utxos_from_addr(['mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd']) +# +# +# if __name__ == '__main__': +# main() diff --git a/client/btc.py b/client/btc.py new file mode 100644 index 0000000..76c400d --- /dev/null +++ b/client/btc.py @@ -0,0 +1,508 @@ +"""Module to support bitcoin operations using a +different codebase than joinmarket's own. +""" +#Protocol constants +BTC_P2PK_VBYTE = {"mainnet": 0x00, "testnet": 0x6f} +BTC_P2SH_VBYTE = {"mainnet": 0x05, "testnet": 0xc4} +PODLE_COMMIT_FILE = None + +from .support import get_log +import binascii, sys, re, hashlib, base64 +from pprint import pformat +log = get_log() + +#Required only for PoDLE calculation: +N = 115792089237316195423570985008687907852837564279074904382605163141518161494337 + +if sys.version_info.major == 2: + string_types = (str, unicode) + string_or_bytes_types = string_types + int_types = (int, float, long) + + # Base switching + code_strings = { + 2: '01', + 10: '0123456789', + 16: '0123456789abcdef', + 32: 'abcdefghijklmnopqrstuvwxyz234567', + 58: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz', + 256: ''.join([chr(x) for x in range(256)]) + } + + def bin_dbl_sha256(s): + bytes_to_hash = from_string_to_bytes(s) + return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest() + + def lpad(msg, symbol, length): + if len(msg) >= length: + return msg + return symbol * (length - len(msg)) + msg + + def get_code_string(base): + if base in code_strings: + return code_strings[base] + else: + raise ValueError("Invalid base!") + + def changebase(string, frm, to, minlen=0): + if frm == to: + return lpad(string, get_code_string(frm)[0], minlen) + return encode(decode(string, frm), to, minlen) + + def bin_to_b58check(inp, magicbyte=0): + inp_fmtd = chr(int(magicbyte)) + inp + leadingzbytes = len(re.match('^\x00*', inp_fmtd).group(0)) + checksum = bin_dbl_sha256(inp_fmtd)[:4] + return '1' * leadingzbytes + changebase(inp_fmtd + checksum, 256, 58) + + def bytes_to_hex_string(b): + return b.encode('hex') + + def safe_from_hex(s): + return s.decode('hex') + + def from_int_to_byte(a): + return chr(a) + + def from_byte_to_int(a): + return ord(a) + + def from_string_to_bytes(a): + return a + + def safe_hexlify(a): + return binascii.hexlify(a) + + def encode(val, base, minlen=0): + base, minlen = int(base), int(minlen) + code_string = get_code_string(base) + result = "" + while val > 0: + result = code_string[val % base] + result + val //= base + return code_string[0] * max(minlen - len(result), 0) + result + + def decode(string, base): + base = int(base) + code_string = get_code_string(base) + result = 0 + if base == 16: + string = string.lower() + while len(string) > 0: + result *= base + result += code_string.find(string[0]) + string = string[1:] + return result + +else: + raise NotImplementedError("Only Python2 currently supported by btc interface") + +interface = "joinmarket-joinmarket" + +try: + from bitcoin import * + bjm = True +except ImportError: + #TODO figure out the right flexibility structure + + interface = "joinmarket-electrum" + + if interface != "joinmarket-electrum": + raise NotImplementedError + + not_supported_string = "not supported by: " + interface + + #Electrum specific code starts here + import electrum.bitcoin as ebt + import electrum.transaction as etr + from ecdsa.ecdsa import curve_secp256k1, generator_secp256k1, point_is_valid + from ecdsa.util import string_to_number, sigdecode_der, sigencode_der + from ecdsa import VerifyingKey, BadSignatureError, BadDigestError + from ecdsa.curves import SECP256k1 + from ecdsa.numbertheory import square_root_mod_prime + from ecdsa.ellipticcurve import Point + + class PPubKey(object): + def __init__(self, serP): + self._point = ebt.ser_to_point(serP) + def serialize(self): + return ebt.point_to_ser(self._point) + + class PPrivKey(object): + def __init__(self, scalar): + self._privkey = ebt.EC_KEY(scalar) + self.private_key = scalar + self.pubkey = PPubKey(binascii.unhexlify( + self._privkey.get_public_key())) + + podle_PublicKey_class = PPubKey + podle_PrivateKey_class = PPrivKey + + def podle_PublicKey(P): + return PPubKey(P) + + def podle_PrivateKey(priv): + return PPrivKey(priv) + + def multiply(s, pub, usehex, rawpub=True, return_serialized=True): + """s should be 32 byte scalar, pub should be of type + podle_PublicKey_class + """ + if usehex: + s = binascii.unhexlify(s) + pub = binascii.unhexlify(pub) + ppub = PPubKey(pub) + p = ppub._point + s_int = decode(s, 256) + m = p * s_int + r = PPubKey(ebt.point_to_ser(m)) + if return_serialized: + return r.serialize() + return r + + def add_pubkeys(pubkeys, usehex): + """Pubkeys should be a list (for compatibility). + """ + #Not supporting more than 2 items for now, not needed. + assert len(pubkeys) == 2 + if usehex: + pubkeys = [binascii.unhexlify(x) for x in pubkeys] + p1pt, p2pt = [ebt.ser_to_point(x) for x in pubkeys] + sumpt = p1pt + p2pt + return ebt.point_to_ser(sumpt) + + def getG(compressed=True): + scalar = "\x00"*31 + "\x01" + return binascii.unhexlify( + ebt.EC_KEY(scalar).get_public_key(compressed=compressed)) + + def sign(tx): + #transaction signing is handled by the wallet for Electrum + raise NotImplementedError("sign " + not_supported_string) + + def get_version_byte(inp): + leadingzbytes = len(re.match('^1*', inp).group(0)) + data = b'\x00' * leadingzbytes + b58check_to_bin(inp) + return ord(data[0]) + + def b58check_to_bin(addr): + return ebt.DecodeBase58Check(addr)[1:] + + def changebase(inp, frm=256, to=58): + """Implementation of base58 (*not* b58check) conversion + only. Used in message channel verifiable nick construction. + Explicitly disabling any other conversion for now. + """ + if not (frm==256 and to==58): + raise NotImplementedError + return ebt.base_encode(inp, 58) + + def address_to_script(addr): + return etr.Transaction.pay_script(ebt.TYPE_ADDRESS, addr) + + def script_to_address(script): + bin_script = binascii.unhexlify(script) + res = etr.get_address_from_output_script(bin_script) + if not res[0] == ebt.TYPE_ADDRESS: + raise ValueError("Invalid script for bitcoin address") + return res[1] + + def bin_dbl_sha256(x): + return ebt.sha256(ebt.sha256(x)) + + def dbl_sha256(x): + return binascii.hexlify(bin_dbl_sha256(x)) + + def verify_tx_input(tx, i, script, sig, pub): + pub, sig, script = (binascii.unhexlify(x) for x in [pub, sig, script]) + t = etr.Transaction(tx) + t.deserialize() + #to prepare for verification (to do the txhash for modtx) + #we need to have the "address" field set in the input. + typ, addr = etr.get_address_from_output_script(script) + if not typ == ebt.TYPE_ADDRESS: + #Don't support non-p2sh, non-p2pkh for now + log.debug("Invalid script") + return False + t.inputs()[i]["address"] = addr + txforsig = etr.Hash(t.tx_for_sig(i).decode('hex')) + ecdsa_pub = get_ecdsa_verifying_key(pub) + if not ecdsa_pub: + return False + try: + verified = ecdsa_pub.verify_digest(sig, txforsig, + sigdecode = sigdecode_der) + except BadSignatureError, BadDigestError: + return False + return True + + def get_ecdsa_verifying_key(pub): + #some shenanigans required to validate a transaction sig; see + #python.ecdsa PR #54. This will be a lot simpler when that's merged. + #https://github.com/warner/python-ecdsa/pull/54/files + if not pub[0] in ["\x02", "\x03"]: + log.debug("Invalid pubkey") + return None + is_even = pub.startswith('\x02') + x = string_to_number(pub[1:]) + order = SECP256k1.order + p = SECP256k1.curve.p() + alpha = (pow(x, 3, p) + (SECP256k1.curve.a() * x) + SECP256k1.curve.b()) % p + beta = square_root_mod_prime(alpha, p) + if is_even == bool(beta & 1): + y = p - beta + else: + y = beta + if not point_is_valid(SECP256k1.generator, x, y): + return None + + point = Point(SECP256k1.curve, x, y, order) + return VerifyingKey.from_public_point(point, SECP256k1, + hashfunc=hashlib.sha256) + + def ecdsa_verify(msg, sig, pub, usehex=True): + sig = base64.b64decode(sig) + if usehex: + pub = binascii.unhexlify(pub) + verif_key = get_ecdsa_verifying_key(pub) + return verif_key.verify_digest(sig, + ebt.Hash(ebt.msg_magic(msg)), + sigdecode = sigdecode_der) + + def ecdsa_sign(msg, priv, usehex=True): + if usehex: + priv = binascii.unhexlify(priv) + compressed = False + if len(priv) == 33 and priv[-1]=="\x01": + compressed = True + signkey = ebt.EC_KEY(priv[:32]) + private_key = ebt.MySigningKey.from_secret_exponent(signkey.secret, + curve=SECP256k1) + sig = private_key.sign_digest_deterministic(ebt.Hash(ebt.msg_magic(msg)), + hashfunc=hashlib.sha256, + sigencode = sigencode_der) + return base64.b64encode(sig) + + def serialize(txobj): + #It is a rather chunky matter to re-use electrum.transaction code + #to do serialization, it has a very different approach. Hence some + #code duplication here with bitcoin-joinmarket. However we use the + #number encoding functions from Electrum. Also, this is always in hex. + o = [] + o.append(ebt.int_to_hex(txobj["version"], 4)) + o.append(ebt.var_int(len(txobj["ins"]))) + for inp in txobj["ins"]: + binhash = binascii.unhexlify(inp["outpoint"]["hash"]) + binhash = binhash[::-1] + o.append(binascii.hexlify(binhash)) + o.append(ebt.int_to_hex(inp["outpoint"]["index"], 4)) + o.append(ebt.var_int(len(inp["script"])/2) + inp["script"]) + o.append(ebt.int_to_hex(inp["sequence"], 4)) + o.append(ebt.var_int(len(txobj["outs"]))) + for out in txobj["outs"]: + o.append(ebt.int_to_hex(out["value"], 8)) + o.append(ebt.var_int(len(out["script"])/2) + out["script"]) + o.append(ebt.int_to_hex(txobj["locktime"], 4)) + return ''.join(o) + + def deserialize_script(scriptSig): + #Assumes P2PKH scriptSig + d = {} + etr.parse_scriptSig(d, binascii.unhexlify(scriptSig)) + return (d["signatures"][0], d["pubkeys"][0]) + + def deserialize(txhex): + t = etr.deserialize(txhex) + #translation from Electrum deserialization + #to pybitcointools form as used in joinmarket + #pybitcointools structure: + #obj = {"ins": [..], "outs": [..], "locktime": int} + #where ins elements are: + #{"outpoint": {"hash": bigendian32,"index": int}, + #"script": hex,"sequence": int} + #and outs elements are: + #{"script": hex, "value": int} + # + #while electrum.transaction.deserialize returns object + #like: + #{"version": int, "inputs": [..], "outputs": [..], "lockTime": int} + obj = {} + obj["version"] = t["version"] + obj["locktime"] = t["lockTime"] + obj["ins"] = [] + obj["outs"] = [] + for i in t["inputs"]: + outpoint = {"hash": i["prevout_hash"], "index": i["prevout_n"]} + scr = i["scriptSig"] + sequence = i["sequence"] + obj["ins"].append({"outpoint": outpoint, "script": scr, "sequence": sequence}) + for i in t["outputs"]: + obj["outs"].append({"script": i["scriptPubKey"], "value": i["value"]}) + return obj + + def privkey_to_pubkey(privkey, usehex=True): + if usehex: + privkey = binascii.unhexlify(privkey) + if len(privkey)==33 and privkey[-1] == "\x01": + compressed = True + privkey = privkey[:32] + elif len(privkey)==32: + compressed=False + else: + raise ValueError("Invalid private key") + sec = ebt.SecretToASecret(privkey, compressed=compressed, + addrtype=BTC_P2PK_VBYTE["mainnet"]) + + retval = ebt.public_key_from_private_key(sec) + if usehex: + return retval + return binascii.unhexlify(retval) + + privtopub = privkey_to_pubkey + + def privkey_to_address(privkey, magicbyte=0): + pubkey = privkey_to_pubkey(privkey) + return pubkey_to_address(pubkey, magicbyte) + + privtoaddr = privkey_to_address + + def pubkey_to_address(pub, magicbyte=0): + h160 = ebt.hash_160(pub.decode('hex')) + return ebt.hash_160_to_bc_address(h160, addrtype=magicbyte) + + pubtoaddr = pubkey_to_address + + def from_wif_privkey(privkey, vbyte=0): + #converts a WIF compressed privkey to a hex private key + return binascii.hexlify(ebt.ASecretToSecret(privkey, addrtype=vbyte)) + + def txhash(txhex): + t = etr.Transaction(txhex) + return t.hash() + + #A simple copy-paste for now; move into support.py perhaps? TODO + def estimate_tx_size(ins, outs, txtype='p2pkh'): + '''Estimate transaction size. + Assuming p2pkh: + out: 8+1+3+2+20=34, in: 1+32+4+1+1+~73+1+1+33=147, + ver:4,seq:4, +2 (len in,out) + total ~= 34*len_out + 147*len_in + 10 (sig sizes vary slightly) + ''' + if txtype == 'p2pkh': + return 10 + ins * 147 + 34 * outs + else: + raise NotImplementedError("Non p2pkh transaction size estimation not" + + "yet implemented") + + def mktx(ins, outs): + #Copy-paste from bitcoin-joinmarket + txobj = {"locktime": 0, "version": 1, "ins": [], "outs": []} + for i in ins: + if isinstance(i, dict) and "outpoint" in i: + txobj["ins"].append(i) + else: + if isinstance(i, dict) and "output" in i: + i = i["output"] + txobj["ins"].append({ + "outpoint": {"hash": i[:64], + "index": int(i[65:])}, + "script": "", + "sequence": 4294967295 + }) + for o in outs: + if not isinstance(o, dict): + addr = o[:o.find(':')] + val = int(o[o.find(':') + 1:]) + o = {} + if re.match('^[0-9a-fA-F]*$', addr): + o["script"] = addr + else: + o["address"] = addr + o["value"] = val + + outobj = {} + if "address" in o: + outobj["script"] = address_to_script(o["address"]) + elif "script" in o: + outobj["script"] = o["script"] + else: + raise Exception("Could not find 'address' or 'script' in output.") + outobj["value"] = o["value"] + txobj["outs"].append(outobj) + + return serialize(txobj) + + def set_commitment_file(file_location): + global PODLE_COMMIT_FILE + PODLE_COMMIT_FILE = file_location + +def test_btc(): + #Sign and verify test (for message signing in joinmarket handshake) + print("Using interface " + interface) + priv = dbl_sha256("hello") + "01" + x = ecdsa_sign("helloxxx", priv) + log.debug("Got: " + x) + y = ecdsa_verify("helloxxx", x, privkey_to_pubkey(priv)) + log.debug("Sig ver: " + str(y)) + assert y + + #address/script conversion test + test_addr = "1LT6rwv26bV7mgvRosoSCyGM7ttVRsYidP" + #Electrum has no support for testnet! + #test_test_addr = "mgvipZr8kX7fZFQU7QsKTCJT9QCfaiswV7" + assert script_to_address(address_to_script(test_addr))==test_addr + assert get_version_byte(test_addr)==0 + + #Transaction creation test. + raw_valid_tx = "01000000064cdfe43ad43b187b738644363144784a09bf6d408012409cf9934591109a789b060000006b483045022100d4309edbb8253e62fb59462f2ff5c3445923e0299bf1a15ac5f7db3da5752bee022066f3f219de7e6ee56c3d600da757ec1051cbd11b42969b8935ae35642b6a2e84012102e94b49525342110266a1dc7651221507318c4cb914ede004b3098650e9b951b6ffffffffc2a9b3e8285c2e7aaee2ea50f792172c920c43a675fa8e8d70976727c8752adf030000006a47304402202763d8ad9e41c99c5af587c69d267493773dc9567519a64db8b707af5daf07f0022011729c6d241ad5abe48687d084644bd442b5f9038db04fb28da674126183aca5012102d2cbeb9386fd201bc6eecf27b2858f7bc27462cd9b43ae464e9ef3281f97a3e0ffffffffa787e89792a93111ff08f5a083234c7c2410bd69b6eef42be0fc5f026a3a1cf0030000006b483045022100c3b86d7acadf1be3d8ea6706daedb842b09732621e830440481370d423703741022009fd0f90a07babd481f1011ec883b2aa248c6a4a433599c5b203c6b93fc03b67012103f9a47d3958281b6749921fdf6d9edde0176342c00ced7caacab9ab3a64795086ffffffff23fb90cebcb1784a7a4a0a35489356ba64cf95c0afdc5a0f0184dc22668ff41f050000006b483045022100ea698e5952e23ffdf6d58bdc73e91c555867e3ad99ac9b583f492882395ace9a0220705abe597972d45923fe0515695dd7b99dcfa50e69d49c03a8126180fd263bc70121036532aa886851548a5b62bff29b4c36bfdc33e68c7dbee8efb4b440e50c5ebc6effffffffd401de8afd8fd323ab6abd9db1d261ac69e7c1d2be7f1a40004e7659b7d6cd9b030000006b483045022100b09c4e7f227f2f86d1965edbc4c92b9058243300f3bc62a3169591aacb60ca4d0220390d0d7ae2ee7dab200e166337c65d4a62b576dc4fa138ce40efd240c57346fc0121034cd59665d736d927d9613c7624f8d616d483b06ab8993446f6119f18e22731feffffffff38b8b3ae5fe9ef09c9f1583c9d6cc128bbd2639d49aca97b7686a74ba91bb32a040000006a4730440220105d93aba953edf008cc5b16ac81c10d97db6e59a3e13062ceef7cc1fbffd2ad022027b14b4162d70c4448bec7cb086b4e52880b51b282de98019ec3038153e25ed0012102cdbfb52b3e164203845f72391a3a58205834a3ad473a9d9878488dc1594aa0d4ffffffff087edb0000000000001976a914a1e5f40c6171e91183533f16bbda35e45182bcfa88ac80d97800000000001976a91482985ea6f877d70692072af967af305005fc86fd88ac80d97800000000001976a914a698b206b9f654974afd2056c85c52f88e4c2b2488ac9970af05000000001976a914b05dbb0ede1191e2871209affd8a5922e0a3275288ac80d97800000000001976a914619b3b22b7b66220d22907b8600724aecc49f03488acabc80000000000001976a914911c8c57eb12aa2c1cdce92f82c7e0405a2f3c6988ac80d97800000000001976a91464cd0ed04862f2b7101e9394285d2b3066e5e4dc88ac13b14100000000001976a9143f81fa4fd890845882fbb5226539d9643c99f0f488ac00000000" + rvtxid = "4489a8cc933cb4e94915ead5b57b4aa707212c1f7b317187b500491e068c7887" + if interface == "joinmarket-electrum": + t = etr.Transaction(raw_valid_tx) + assert rvtxid == t.hash() + + #Transaction deserialization/serialization test + #Electrum requires this call to fill out Transactionfields + t.deserialize() + #log.debug("Got inputs: " + str(t.inputs)) + ourdeser = deserialize(t.raw) + ourraw = serialize(ourdeser) + #log.debug("Recreated: \n" + ourraw) + assert ourraw == raw_valid_tx + #double check round trip too + assert deserialize(ourraw) == ourdeser + txinslist = t.inputs() + elif interface == "joinmarket-joinmarket": + assert serialize(deserialize(raw_valid_tx)) == raw_valid_tx + t = deserialize(raw_valid_tx) + txinslist = t["ins"] + else: + raise NotImplementedError("No such interface?") + + #Transaction signature verification tests. + #All currently assuming 100% p2pkh. + for i, tin in enumerate(txinslist): + if interface == "joinmarket-electrum": + script = address_to_script(tin["address"]) + sig = tin["signatures"][0] + pub = tin["pubkeys"][0] + elif interface == "joinmarket-joinmarket": + log.debug("Joinmarket working with this script: " + tin["script"]) + scriptSig = tin["script"] + #We need to parse out the pubkey, convert to address, then convert + #to a pubkeyscript; this assumes p2pkh. Note that this is handled + #internally by the joinmarket blockchain/maker/taker code, so only + #for tests. + pub = scriptSig[-66:] + script = address_to_script(pubkey_to_address(pub)) + log.debug("Converted to this addr script: " + script) + #drop the length bytes from the start of sig and pub + sig = scriptSig[2:-68] + else: + raise NotImplementedError("No such interface?") + log.debug("Got sig, script, pub: " + " ".join([sig, script, pub])) + assert verify_tx_input(raw_valid_tx, i, script, sig, pub) + log.debug("Sig at: " + str(i) + " OK.") + + #Note there are no transaction signing tests, as + #this is done by the wallet in this interface. + log.debug("All tests passed.") \ No newline at end of file diff --git a/client/client_protocol.py b/client/client_protocol.py new file mode 100644 index 0000000..88ebbcd --- /dev/null +++ b/client/client_protocol.py @@ -0,0 +1,230 @@ +#! /usr/bin/env python +from __future__ import print_function +from twisted.python.log import startLogging, err +from twisted.internet import protocol, reactor +from twisted.protocols import amp +from twisted.internet.protocol import ClientFactory +from twisted.internet.endpoints import TCP4ClientEndpoint +import commands +from sys import stdout + +import json +import random +import string +import time +import hashlib +import os +from joinmarketclient import (Taker, Wallet, jm_single, get_irc_mchannels, + load_program_config, get_log) + +import btc + +jlog = get_log() + +class JMProtocolError(Exception): + pass + +class JMTakerClientProtocol(amp.AMP): + + def __init__(self, factory, taker, nick_priv=None): + self.taker = taker + self.factory = factory + self.orderbook = None + self.supported_messages = ["JM_UP", "JM_SETUP_DONE", "JM_FILL_RESPONSE", + "JM_OFFERS", "JM_SIG_RECEIVED", + "JM_REQUEST_MSGSIG", + "JM_REQUEST_MSGSIG_VERIFY", "JM_INIT_PROTO"] + if not nick_priv: + self.nick_priv = hashlib.sha256(os.urandom(16)).hexdigest() + '01' + else: + self.nick_priv = nick_priv + + def checkClientResponse(self, response): + """A generic check of client acceptance; any failure + is considered criticial. + """ + if 'accepted' not in response or not response['accepted']: + reactor.stop() + + def connectionMade(self): + """Upon confirmation of network connection + to daemon, request message channel initialization + with relevant config data for our message channels + """ + #needed only for channel naming convention + blockchain_source = jm_single().config.get("BLOCKCHAIN", + "blockchain_source") + network = jm_single().config.get("BLOCKCHAIN", "network") + irc_configs = get_irc_mchannels() + minmakers = jm_single().config.getint("POLICY", "minimum_makers") + maker_timeout_sec = jm_single().maker_timeout_sec + d = self.callRemote(commands.JMInit, + bcsource=blockchain_source, + network=network, + irc_configs=json.dumps(irc_configs), + minmakers=minmakers, + maker_timeout_sec=maker_timeout_sec) + d.addCallback(self.checkClientResponse) + + def send_data(self, cmd, data): + JMProtocol.send_data(self, cmd, data) + + def set_nick(self): + self.nick_pubkey = btc.privtopub(self.nick_priv) + self.nick_pkh_raw = hashlib.sha256(self.nick_pubkey).digest()[ + :self.nick_hashlen] + self.nick_pkh = btc.changebase(self.nick_pkh_raw, 256, 58) + #right pad to maximum possible; b58 is not fixed length. + #Use 'O' as one of the 4 not included chars in base58. + self.nick_pkh += 'O' * (self.nick_maxencoded - len(self.nick_pkh)) + #The constructed length will be 1 + 1 + NICK_MAX_ENCODED + self.nick = self.nick_header + str(self.jm_version) + self.nick_pkh + jm_single().nickname = self.nick + + @commands.JMInitProto.responder + def on_JM_INIT_PROTO(self, nick_hash_length, nick_max_encoded, + joinmarket_nick_header, joinmarket_version): + """Daemon indicates init-ed status and passes back protocol constants. + Use protocol settings to set actual nick from nick private key, + then call setup to instantiate message channel connections in the daemon. + """ + self.nick_hashlen = nick_hash_length + self.nick_maxencoded = nick_max_encoded + self.nick_header = joinmarket_nick_header + self.jm_version = joinmarket_version + self.set_nick() + d = self.callRemote(commands.JMStartMC, + nick=self.nick) + d.addCallback(self.checkClientResponse) + return {'accepted': True} + + @commands.JMUp.responder + def on_JM_UP(self): + d = self.callRemote(commands.JMSetup, + role="TAKER", + n_counterparties=4) #TODO this number should be set + d.addCallback(self.checkClientResponse) + return {'accepted': True} + + @commands.JMSetupDone.responder + def on_JM_SETUP_DONE(self): + jlog.info("JM daemon setup complete") + #The daemon is ready and has requested the orderbook + #from the pit; we can request the entire orderbook + #and filter it as we choose. + reactor.callLater(jm_single().maker_timeout_sec, self.get_offers) + return {'accepted': True} + + @commands.JMFillResponse.responder + def on_JM_FILL_RESPONSE(self, success, ioauth_data): + """Receives the entire set of phase 1 data (principally utxos) + from the counterparties and passes through to the Taker for + tx construction, if successful. Then passes back the phase 2 + initiating data to the daemon. + """ + ioauth_data = json.loads(ioauth_data) + if not success: + jlog.info("Makers didnt respond blah blah") + return {'accepted': True} + else: + jlog.info("Makers responded with: " + json.dumps(ioauth_data)) + retval = self.taker.receive_utxos(ioauth_data) + if not retval[0]: + jlog.info("Taker is not continuing, phase 2 abandoned.") + jlog.info("Reason: " + str(retval[1])) + return {'accepted': False} + else: + nick_list, txhex = retval[1:] + reactor.callLater(0, self.make_tx, nick_list, txhex) + return {'accepted': True} + + @commands.JMOffers.responder + def on_JM_OFFERS(self, orderbook): + self.orderbook = json.loads(orderbook) + jlog.info("Got the orderbook: " + str(self.orderbook)) + retval = self.taker.initialize(self.orderbook) + #format of retval is: + #True, self.cjamount, commitment, revelation, self.filtered_orderbook) + if not retval[0]: + jlog.info("Taker not continuing after receipt of orderbook") + return + amt, cmt, rev, foffers = retval[1:] + d = self.callRemote(commands.JMFill, + amount=amt, + commitment=str(cmt), + revelation=str(rev), + filled_offers=json.dumps(foffers)) + d.addCallback(self.checkClientResponse) + return {'accepted': True} + + @commands.JMSigReceived.responder + def on_JM_SIG_RECEIVED(self, nick, sig): + retval = self.taker.on_sig(nick, sig) + if retval: + #flag indicating completion; but Taker + #handles tx pushing, just update state + self.state = 4 + return {'accepted': True} + + @commands.JMRequestMsgSig.responder + def on_JM_REQUEST_MSGSIG(self, nick, cmd, msg, msg_to_be_signed, hostid): + sig = btc.ecdsa_sign(str(msg_to_be_signed), self.nick_priv) + msg_to_return = str(msg) + " " + self.nick_pubkey + " " + sig + d = self.callRemote(commands.JMMsgSignature, + nick=nick, + cmd=cmd, + msg_to_return=msg_to_return, + hostid=hostid) + d.addCallback(self.checkClientResponse) + return {'accepted': True} + + @commands.JMRequestMsgSigVerify.responder + def on_JM_REQUEST_MSGSIG_VERIFY(self, msg, fullmsg, sig, pubkey, nick, + hashlen, max_encoded, hostid): + jlog.info("Got a request to verify a signature") + verif_result = True + if not btc.ecdsa_verify(str(msg), sig, pubkey): + jlog.debug("nick signature verification failed, ignoring.") + verif_result = False + #check that nick matches hash of pubkey + nick_pkh_raw = hashlib.sha256(pubkey).digest()[:hashlen] + nick_stripped = nick[2:2 + max_encoded] + #strip right padding + nick_unpadded = ''.join([x for x in nick_stripped if x != 'O']) + if not nick_unpadded == btc.changebase(nick_pkh_raw, 256, 58): + jlog.debug("Nick hash check failed, expected: " + str(nick_unpadded) + + ", got: " + str(btc.changebase(nick_pkh_raw, 256, 58))) + verif_result = False + jlog.info("Sending a verifcation result: " + str(verif_result)) + d = self.callRemote(commands.JMMsgSignatureVerify, + verif_result=verif_result, + nick=nick, + fullmsg=fullmsg, + hostid=hostid) + d.addCallback(self.checkClientResponse) + return {'accepted': True} + + def get_offers(self): + d = self.callRemote(commands.JMRequestOffers) + d.addCallback(self.checkClientResponse) + + def make_tx(self, nick_list, txhex): + d = self.callRemote(commands.JMMakeTx, + nick_list= json.dumps(nick_list), + txhex=txhex) + d.addCallback(self.checkClientResponse) + + +class JMTakerClientProtocolFactory(protocol.ClientFactory): + protocol = JMTakerClientProtocol + + def __init__(self, taker): + self.taker = taker + + def buildProtocol(self, addr): + return JMTakerClientProtocol(self, self.taker) + + +def start_reactor(host, port, factory, ish=True): + reactor.connectTCP(host, port, factory) + reactor.run(installSignalHandlers=ish) diff --git a/client/commands.py b/client/commands.py new file mode 100644 index 0000000..7dfe709 --- /dev/null +++ b/client/commands.py @@ -0,0 +1,92 @@ +from twisted.protocols.amp import Integer, String, Unicode, Boolean, Command + +class DaemonNotReady(Exception): + pass + +class JMCommand(Command): + #a default response type + response = [('accepted', Boolean())] + +#commands from client to daemon + +class JMInit(JMCommand): + arguments = [('bcsource', String()), + ('network', String()), + ('irc_configs', String()), + ('minmakers', Integer()), + ('maker_timeout_sec', Integer())] + errors = {DaemonNotReady: 'daemon is not ready'} + +class JMStartMC(JMCommand): + arguments = [('nick', String())] + +class JMSetup(JMCommand): + arguments = [('role', String()), + ('n_counterparties', Integer())] + +class JMRequestOffers(JMCommand): + arguments = [] + +class JMFill(JMCommand): + arguments = [('amount', Integer()), + ('commitment', String()), + ('revelation', String()), + ('filled_offers', String())] + +class JMMakeTx(JMCommand): + arguments = [('nick_list', String()), + ('txhex', String())] + +class JMMsgSignature(JMCommand): + arguments = [('nick', String()), + ('cmd', String()), + ('msg_to_return', String()), + ('hostid', String())] + +class JMMsgSignatureVerify(JMCommand): + arguments = [('verif_result', Boolean()), + ('nick', String()), + ('fullmsg', String()), + ('hostid', String())] + +#commands from daemon to client + +class JMInitProto(JMCommand): + arguments = [('nick_hash_length', Integer()), + ('nick_max_encoded', Integer()), + ('joinmarket_nick_header', String()), + ('joinmarket_version', Integer())] + +class JMUp(JMCommand): + arguments = [] + +class JMSetupDone(JMCommand): + arguments = [] + +class JMOffers(JMCommand): + arguments = [('orderbook', String())] + +class JMFillResponse(JMCommand): + arguments = [('success', Boolean()), + ('ioauth_data', String())] + +class JMSigReceived(JMCommand): + arguments = [('nick', String()), + ('sig', String())] + +class JMRequestMsgSig(JMCommand): + arguments = [('nick', String()), + ('cmd', String()), + ('msg', String()), + ('msg_to_be_signed', String()), + ('hostid', String())] + +class JMRequestMsgSigVerify(JMCommand): + arguments = [('msg', String()), + ('fullmsg', String()), + ('sig', String()), + ('pubkey', String()), + ('nick', String()), + ('hashlen', Integer()), + ('max_encoded', Integer()), + ('hostid', String())] \ No newline at end of file diff --git a/client/configure.py b/client/configure.py new file mode 100644 index 0000000..63392b3 --- /dev/null +++ b/client/configure.py @@ -0,0 +1,376 @@ +from __future__ import print_function + +import io +import logging +import threading +import os +import binascii +import sys + +from ConfigParser import SafeConfigParser, NoOptionError + +import btc +from joinmarketclient.jsonrpc import JsonRpc +from joinmarketclient.support import get_log, joinmarket_alert, core_alert, debug_silence +from joinmarketclient.podle import set_commitment_file + +log = get_log() + + +class AttributeDict(object): + """ + A class to convert a nested Dictionary into an object with key-values + accessibly using attribute notation (AttributeDict.attribute) instead of + key notation (Dict["key"]). This class recursively sets Dicts to objects, + allowing you to recurse down nested dicts (like: AttributeDict.attr.attr) + """ + + def __init__(self, **entries): + self.add_entries(**entries) + + def add_entries(self, **entries): + for key, value in entries.items(): + if type(value) is dict: + self.__dict__[key] = AttributeDict(**value) + else: + self.__dict__[key] = value + + def __setattr__(self, name, value): + if name == 'nickname' and value: + logFormatter = logging.Formatter( + ('%(asctime)s [%(threadName)-12.12s] ' + '[%(levelname)-5.5s] %(message)s')) + logsdir = os.path.join(os.path.dirname( + global_singleton.config_location), "logs") + fileHandler = logging.FileHandler( + logsdir + '/{}.log'.format(value)) + fileHandler.setFormatter(logFormatter) + log.addHandler(fileHandler) + + super(AttributeDict, self).__setattr__(name, value) + + def __getitem__(self, key): + """ + Provides dict-style access to attributes + """ + return getattr(self, key) + + +global_singleton = AttributeDict() +global_singleton.JM_VERSION = 5 +global_singleton.nickname = None +global_singleton.BITCOIN_DUST_THRESHOLD = 2730 +global_singleton.DUST_THRESHOLD = 10 * global_singleton.BITCOIN_DUST_THRESHOLD +global_singleton.bc_interface = None +global_singleton.maker_timeout_sec = 60 +global_singleton.debug_file_lock = threading.Lock() +global_singleton.ordername_list = ["reloffer", "absoffer"] +global_singleton.debug_file_handle = None +global_singleton.blacklist_file_lock = threading.Lock() +global_singleton.core_alert = core_alert +global_singleton.joinmarket_alert = joinmarket_alert +global_singleton.debug_silence = debug_silence +global_singleton.config = SafeConfigParser() +#This is reset to a full path after load_program_config call +global_singleton.config_location = 'joinmarket.cfg' +#as above +global_singleton.commit_file_location = 'cmttools/commitments.json' +global_singleton.wait_for_commitments = 0 + + +def jm_single(): + return global_singleton + +# FIXME: Add rpc_* options here in the future! +required_options = {'BLOCKCHAIN': ['blockchain_source', 'network'], + 'MESSAGING': ['host', 'channel', 'port'], + 'POLICY': ['absurd_fee_per_kb', 'taker_utxo_retries', + 'taker_utxo_age', 'taker_utxo_amtpercent']} + +defaultconfig = \ + """ +[BLOCKCHAIN] +blockchain_source = blockr +#options: blockr, bitcoin-rpc, regtest +# for instructions on bitcoin-rpc read +# https://github.com/chris-belcher/joinmarket/wiki/Running-JoinMarket-with-Bitcoin-Core-full-node +network = mainnet +rpc_host = localhost +rpc_port = 8332 +rpc_user = bitcoin +rpc_password = password + +[MESSAGING] +host = irc.cyberguerrilla.org +channel = joinmarket-pit +port = 6697 +usessl = true +socks5 = false +socks5_host = localhost +socks5_port = 9050 +#for tor +#host = 6dvj6v5imhny3anf.onion +#onion / i2p have their own ports on CGAN +#port = 6698 +#usessl = true +#socks5 = true + +[TIMEOUT] +maker_timeout_sec = 30 +unconfirm_timeout_sec = 90 +confirm_timeout_hours = 6 + +[POLICY] +# for dust sweeping, try merge_algorithm = gradual +# for more rapid dust sweeping, try merge_algorithm = greedy +# for most rapid dust sweeping, try merge_algorithm = greediest +# but don't forget to bump your miner fees! +merge_algorithm = default +# the fee estimate is based on a projection of how many satoshis +# per kB are needed to get in one of the next N blocks, N set here +# as the value of 'tx_fees'. This estimate is high if you set N=1, +# so we choose N=3 for a more reasonable figure, +# as our default. Note that for clients not using a local blockchain +# instance, we retrieve an estimate from the API at blockcypher.com, currently. +tx_fees = 3 +# For users getting transaction fee estimates over an API +# (currently blockcypher, could be others), place a sanity +# check limit on the satoshis-per-kB to be paid. This limit +# is also applied to users using Core, even though Core has its +# own sanity check limit, which is currently 1,000,000 satoshis. +absurd_fee_per_kb = 150000 +# the range of confirmations passed to the `listunspent` bitcoind RPC call +# 1st value is the inclusive minimum, defaults to one confirmation +# 2nd value is the exclusive maximum, defaults to most-positive-bignum (Google Me!) +# leaving it unset or empty defers to bitcoind's default values, ie [1, 9999999] +#listunspent_args = [] +# that's what you should do, unless you have a specific reason, eg: +# !!! WARNING !!! CONFIGURING THIS WHILE TAKING LIQUIDITY FROM +# !!! WARNING !!! THE PUBLIC ORDERBOOK LEAKS YOUR INPUT MERGES +# spend from unconfirmed transactions: listunspent_args = [0] +# display only unconfirmed transactions: listunspent_args = [0, 1] +# defend against small reorganizations: listunspent_args = [3] +# who is at risk of reorganization?: listunspent_args = [0, 2] +# NB: using 0 for the 1st value with scripts other than wallet-tool could cause +# spends from unconfirmed inputs, which may then get malleated or double-spent! +# other counterparties are likely to reject unconfirmed inputs... don't do it. + +#options: self, random-peer, not-self, random-maker +# self = broadcast transaction with your own ip +# random-peer = everyone who took part in the coinjoin has a chance of broadcasting +# not-self = never broadcast with your own ip +# random-maker = every peer on joinmarket has a chance of broadcasting, including yourself +tx_broadcast = self +minimum_makers = 2 +#THE FOLLOWING SETTINGS ARE REQUIRED TO DEFEND AGAINST SNOOPERS. +#DON'T ALTER THEM UNLESS YOU UNDERSTAND THE IMPLICATIONS. + +# number of retries allowed for a specific utxo, to prevent DOS/snooping. +# Lower settings make snooping more expensive, but also prevent honest users +# from retrying if an error occurs. +taker_utxo_retries = 3 + +# number of confirmations required for the commitment utxo mentioned above. +# this effectively rate-limits a snooper. +taker_utxo_age = 5 + +# percentage of coinjoin amount that the commitment utxo must have +# as a minimum BTC amount. Thus 20 means a 1BTC coinjoin requires the +# utxo to be at least 0.2 btc. +taker_utxo_amtpercent = 20 + +#Set to 1 to accept broadcast PoDLE commitments from other bots, and +#add them to your blacklist (only relevant for Makers). +#There is no way to spoof these values, so the only "risk" is that +#someone fills your blacklist file with a lot of data. +accept_commitment_broadcasts = 1 + +#Location of your commitments.json file (stores commitments you've used +#and those you want to use in future), relative to root joinmarket directory. +commit_file_location = cmttools/commitments.json +""" + + +def get_irc_mchannels(): + fields = [("host", str), ("port", int), ("channel", str), ("usessl", str), + ("socks5", str), ("socks5_host", str), ("socks5_port", str)] + configdata = {} + for f, t in fields: + vals = jm_single().config.get("MESSAGING", f).split(",") + if t == str: + vals = [x.strip() for x in vals] + else: + vals = [t(x) for x in vals] + configdata[f] = vals + configs = [] + for i in range(len(configdata['host'])): + newconfig = dict([(x, configdata[x][i]) for x in configdata]) + newconfig['btcnet'] = get_network() + configs.append(newconfig) + return configs + + +def get_config_irc_channel(channel_name): + channel = "#" + channel_name + if get_network() == 'testnet': + channel += '-test' + return channel + + +def get_network(): + """Returns network name""" + return global_singleton.config.get("BLOCKCHAIN", "network") + + +def get_p2sh_vbyte(): + return btc.BTC_P2SH_VBYTE[get_network()] + + +def get_p2pk_vbyte(): + return btc.BTC_P2PK_VBYTE[get_network()] + + +def validate_address(addr): + try: + ver = btc.get_version_byte(addr) + except AssertionError: + return False, 'Checksum wrong. Typo in address?' + except Exception: + return False, "Invalid bitcoin address" + if ver != get_p2pk_vbyte() and ver != get_p2sh_vbyte(): + return False, 'Wrong address version. Testnet/mainnet confused?' + if len(btc.b58check_to_bin(addr)) != 20: + return False, "Address has correct checksum but wrong length." + return True, 'address validated' + + +def donation_address(reusable_donation_pubkey=None): + if not reusable_donation_pubkey: + reusable_donation_pubkey = ('02be838257fbfddabaea03afbb9f16e852' + '9dfe2de921260a5c46036d97b5eacf2a') + sign_k = binascii.hexlify(os.urandom(32)) + c = btc.sha256(btc.multiply(sign_k, reusable_donation_pubkey, True)) + sender_pubkey = btc.add_pubkeys( + [reusable_donation_pubkey, btc.privtopub(c + '01', True)], True) + sender_address = btc.pubtoaddr(sender_pubkey, get_p2pk_vbyte()) + log.debug('sending coins to ' + sender_address) + return sender_address, sign_k + + +def check_utxo_blacklist(commitment, persist=False): + """Compare a given commitment (H(P2) for PoDLE) + with the persisted blacklist log file; + if it has been used before, return False (disallowed), + else return True. + If flagged, persist the usage of this commitment to the blacklist file. + """ + #TODO format error checking? + fname = "blacklist" + if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == 'regtest': + fname += "_" + jm_single().nickname + with jm_single().blacklist_file_lock: + if os.path.isfile(fname): + with open(fname, "rb") as f: + blacklisted_commitments = [x.strip() for x in f.readlines()] + else: + blacklisted_commitments = [] + if commitment in blacklisted_commitments: + return False + elif persist: + blacklisted_commitments += [commitment] + with open(fname, "wb") as f: + f.write('\n'.join(blacklisted_commitments)) + f.flush() + #If the commitment is new and we are *not* persisting, nothing to do + #(we only add it to the list on sending io_auth, which represents actual + #usage). + return True + + +def load_program_config(config_path=None, bs=None): + global_singleton.config.readfp(io.BytesIO(defaultconfig)) + if not config_path: + config_path = os.getcwd() + global_singleton.config_location = os.path.join( + config_path, global_singleton.config_location) + loadedFiles = global_singleton.config.read([global_singleton.config_location + ]) + #Hack required for electrum; must be able to enforce a different + #blockchain interface even in default/new load. + if bs: + global_singleton.config.set("BLOCKCHAIN", "blockchain_source", bs) + # Create default config file if not found + if len(loadedFiles) != 1: + with open(global_singleton.config_location, "w") as configfile: + configfile.write(defaultconfig) + + # check for sections + for s in required_options: + if s not in global_singleton.config.sections(): + raise Exception( + "Config file does not contain the required section: " + s) + # then check for specific options + for k, v in required_options.iteritems(): + for o in v: + if o not in global_singleton.config.options(k): + raise Exception( + "Config file does not contain the required option: " + o) + + try: + global_singleton.maker_timeout_sec = global_singleton.config.getint( + 'TIMEOUT', 'maker_timeout_sec') + except NoOptionError: + log.debug('TIMEOUT/maker_timeout_sec not found in .cfg file, ' + 'using default value') + + # configure the interface to the blockchain on startup + global_singleton.bc_interface = get_blockchain_interface_instance( + global_singleton.config) + + #set the location of the commitments file + try: + global_singleton.commit_file_location = global_singleton.config.get( + "POLICY", "commit_file_location") + except NoOptionError: + log.debug("No commitment file location in config, using default " + "location cmttools/commitments.json") + set_commitment_file(os.path.join(config_path, + global_singleton.commit_file_location)) + + +def get_blockchain_interface_instance(_config): + # todo: refactor joinmarket module to get rid of loops + # importing here is necessary to avoid import loops + from joinmarketclient.blockchaininterface import BitcoinCoreInterface, \ + RegtestBitcoinCoreInterface, BlockrInterface, ElectrumWalletInterface + from joinmarketclient.blockchaininterface import CliJsonRpc + + source = _config.get("BLOCKCHAIN", "blockchain_source") + network = get_network() + testnet = network == 'testnet' + if source == 'bitcoin-rpc': + rpc_host = _config.get("BLOCKCHAIN", "rpc_host") + rpc_port = _config.get("BLOCKCHAIN", "rpc_port") + rpc_user = _config.get("BLOCKCHAIN", "rpc_user") + rpc_password = _config.get("BLOCKCHAIN", "rpc_password") + rpc = JsonRpc(rpc_host, rpc_port, rpc_user, rpc_password) + bc_interface = BitcoinCoreInterface(rpc, network) + elif source == 'json-rpc': + bitcoin_cli_cmd = _config.get("BLOCKCHAIN", + "bitcoin_cli_cmd").split(' ') + rpc = CliJsonRpc(bitcoin_cli_cmd, testnet) + bc_interface = BitcoinCoreInterface(rpc, network) + elif source == 'regtest': + rpc_host = _config.get("BLOCKCHAIN", "rpc_host") + rpc_port = _config.get("BLOCKCHAIN", "rpc_port") + rpc_user = _config.get("BLOCKCHAIN", "rpc_user") + rpc_password = _config.get("BLOCKCHAIN", "rpc_password") + rpc = JsonRpc(rpc_host, rpc_port, rpc_user, rpc_password) + bc_interface = RegtestBitcoinCoreInterface(rpc) + elif source == 'blockr': + bc_interface = BlockrInterface(testnet) + elif source == 'electrum': + bc_interface = ElectrumWalletInterface(testnet) + else: + raise ValueError("Invalid blockchain source") + return bc_interface diff --git a/client/jsonrpc.py b/client/jsonrpc.py new file mode 100644 index 0000000..d12243f --- /dev/null +++ b/client/jsonrpc.py @@ -0,0 +1,119 @@ +from __future__ import absolute_import, print_function +# Copyright (C) 2013,2015 by Daniel Kraft +# Copyright (C) 2014 by phelix / blockchained.com +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import base64 +import httplib +import json + + +class JsonRpcError(Exception): + """ + The called method returned an error in the JSON-RPC response. + """ + + def __init__(self, obj): + self.code = obj["code"] + self.message = obj["message"] + + +class JsonRpcConnectionError(Exception): + """ + Error thrown when the RPC connection itself failed. This means + that the server is either down or the connection settings + are wrong. + """ + + pass + + +class JsonRpc(object): + """ + Simple implementation of a JSON-RPC client that is used + to connect to Bitcoin. + """ + + def __init__(self, host, port, user, password): + self.host = host + self.port = port + self.authstr = "%s:%s" % (user, password) + + self.queryId = 1 + + def queryHTTP(self, obj): + """ + Send an appropriate HTTP query to the server. The JSON-RPC + request should be (as object) in 'obj'. If the call succeeds, + the resulting JSON object is returned. In case of an error + with the connection (not JSON-RPC itself), an exception is raised. + """ + + headers = {"User-Agent": "joinmarket", + "Content-Type": "application/json", + "Accept": "application/json"} + headers["Authorization"] = "Basic %s" % base64.b64encode(self.authstr) + + body = json.dumps(obj) + + try: + conn = httplib.HTTPConnection(self.host, self.port) + conn.request("POST", "", body, headers) + response = conn.getresponse() + + if response.status == 401: + conn.close() + raise JsonRpcConnectionError( + "authentication for JSON-RPC failed") + + # All of the codes below are 'fine' from a JSON-RPC point of view. + if response.status not in [200, 404, 500]: + conn.close() + raise JsonRpcConnectionError("unknown error in JSON-RPC") + + data = response.read() + conn.close() + + return json.loads(data) + + except JsonRpcConnectionError as exc: + raise exc + except Exception as exc: + raise JsonRpcConnectionError("JSON-RPC connection failed. Err:" + + repr(exc)) + + def call(self, method, params): + """ + Call a method over JSON-RPC. + """ + + currentId = self.queryId + self.queryId += 1 + + request = {"method": method, "params": params, "id": currentId} + response = self.queryHTTP(request) + + if response["id"] != currentId: + raise JsonRpcConnectionError("invalid id returned by query") + + if response["error"] is not None: + raise JsonRpcError(response["error"]) + + return response["result"] diff --git a/client/old_mnemonic.py b/client/old_mnemonic.py new file mode 100644 index 0000000..32d6761 --- /dev/null +++ b/client/old_mnemonic.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python +# +# Electrum - lightweight Bitcoin client +# Copyright (C) 2011 thomasv@gitorious +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# list of words from http://en.wiktionary.org/wiki/Wiktionary:Frequency_lists/Contemporary_poetry + +words = [ + "like", "just", "love", "know", "never", "want", "time", "out", "there", + "make", "look", "eye", "down", "only", "think", "heart", "back", "then", + "into", "about", "more", "away", "still", "them", "take", "thing", "even", + "through", "long", "always", "world", "too", "friend", "tell", "try", + "hand", "thought", "over", "here", "other", "need", "smile", "again", + "much", "cry", "been", "night", "ever", "little", "said", "end", "some", + "those", "around", "mind", "people", "girl", "leave", "dream", "left", + "turn", "myself", "give", "nothing", "really", "off", "before", "something", + "find", "walk", "wish", "good", "once", "place", "ask", "stop", "keep", + "watch", "seem", "everything", "wait", "got", "yet", "made", "remember", + "start", "alone", "run", "hope", "maybe", "believe", "body", "hate", + "after", "close", "talk", "stand", "own", "each", "hurt", "help", "home", + "god", "soul", "new", "many", "two", "inside", "should", "true", "first", + "fear", "mean", "better", "play", "another", "gone", "change", "use", + "wonder", "someone", "hair", "cold", "open", "best", "any", "behind", + "happen", "water", "dark", "laugh", "stay", "forever", "name", "work", + "show", "sky", "break", "came", "deep", "door", "put", "black", "together", + "upon", "happy", "such", "great", "white", "matter", "fill", "past", + "please", "burn", "cause", "enough", "touch", "moment", "soon", "voice", + "scream", "anything", "stare", "sound", "red", "everyone", "hide", "kiss", + "truth", "death", "beautiful", "mine", "blood", "broken", "very", "pass", + "next", "forget", "tree", "wrong", "air", "mother", "understand", "lip", + "hit", "wall", "memory", "sleep", "free", "high", "realize", "school", + "might", "skin", "sweet", "perfect", "blue", "kill", "breath", "dance", + "against", "fly", "between", "grow", "strong", "under", "listen", "bring", + "sometimes", "speak", "pull", "person", "become", "family", "begin", + "ground", "real", "small", "father", "sure", "feet", "rest", "young", + "finally", "land", "across", "today", "different", "guy", "line", "fire", + "reason", "reach", "second", "slowly", "write", "eat", "smell", "mouth", + "step", "learn", "three", "floor", "promise", "breathe", "darkness", "push", + "earth", "guess", "save", "song", "above", "along", "both", "color", + "house", "almost", "sorry", "anymore", "brother", "okay", "dear", "game", + "fade", "already", "apart", "warm", "beauty", "heard", "notice", "question", + "shine", "began", "piece", "whole", "shadow", "secret", "street", "within", + "finger", "point", "morning", "whisper", "child", "moon", "green", "story", + "glass", "kid", "silence", "since", "soft", "yourself", "empty", "shall", + "angel", "answer", "baby", "bright", "dad", "path", "worry", "hour", "drop", + "follow", "power", "war", "half", "flow", "heaven", "act", "chance", "fact", + "least", "tired", "children", "near", "quite", "afraid", "rise", "sea", + "taste", "window", "cover", "nice", "trust", "lot", "sad", "cool", "force", + "peace", "return", "blind", "easy", "ready", "roll", "rose", "drive", + "held", "music", "beneath", "hang", "mom", "paint", "emotion", "quiet", + "clear", "cloud", "few", "pretty", "bird", "outside", "paper", "picture", + "front", "rock", "simple", "anyone", "meant", "reality", "road", "sense", + "waste", "bit", "leaf", "thank", "happiness", "meet", "men", "smoke", + "truly", "decide", "self", "age", "book", "form", "alive", "carry", + "escape", "damn", "instead", "able", "ice", "minute", "throw", "catch", + "leg", "ring", "course", "goodbye", "lead", "poem", "sick", "corner", + "desire", "known", "problem", "remind", "shoulder", "suppose", "toward", + "wave", "drink", "jump", "woman", "pretend", "sister", "week", "human", + "joy", "crack", "grey", "pray", "surprise", "dry", "knee", "less", "search", + "bleed", "caught", "clean", "embrace", "future", "king", "son", "sorrow", + "chest", "hug", "remain", "sat", "worth", "blow", "daddy", "final", + "parent", "tight", "also", "create", "lonely", "safe", "cross", "dress", + "evil", "silent", "bone", "fate", "perhaps", "anger", "class", "scar", + "snow", "tiny", "tonight", "continue", "control", "dog", "edge", "mirror", + "month", "suddenly", "comfort", "given", "loud", "quickly", "gaze", "plan", + "rush", "stone", "town", "battle", "ignore", "spirit", "stood", "stupid", + "yours", "brown", "build", "dust", "hey", "kept", "pay", "phone", "twist", + "although", "ball", "beyond", "hidden", "nose", "taken", "fail", "float", + "pure", "somehow", "wash", "wrap", "angry", "cheek", "creature", + "forgotten", "heat", "rip", "single", "space", "special", "weak", + "whatever", "yell", "anyway", "blame", "job", "choose", "country", "curse", + "drift", "echo", "figure", "grew", "laughter", "neck", "suffer", "worse", + "yeah", "disappear", "foot", "forward", "knife", "mess", "somewhere", + "stomach", "storm", "beg", "idea", "lift", "offer", "breeze", "field", + "five", "often", "simply", "stuck", "win", "allow", "confuse", "enjoy", + "except", "flower", "seek", "strength", "calm", "grin", "gun", "heavy", + "hill", "large", "ocean", "shoe", "sigh", "straight", "summer", "tongue", + "accept", "crazy", "everyday", "exist", "grass", "mistake", "sent", "shut", + "surround", "table", "ache", "brain", "destroy", "heal", "nature", "shout", + "sign", "stain", "choice", "doubt", "glance", "glow", "mountain", "queen", + "stranger", "throat", "tomorrow", "city", "either", "fish", "flame", + "rather", "shape", "spin", "spread", "ash", "distance", "finish", "image", + "imagine", "important", "nobody", "shatter", "warmth", "became", "feed", + "flesh", "funny", "lust", "shirt", "trouble", "yellow", "attention", "bare", + "bite", "money", "protect", "amaze", "appear", "born", "choke", + "completely", "daughter", "fresh", "friendship", "gentle", "probably", + "six", "deserve", "expect", "grab", "middle", "nightmare", "river", + "thousand", "weight", "worst", "wound", "barely", "bottle", "cream", + "regret", "relationship", "stick", "test", "crush", "endless", "fault", + "itself", "rule", "spill", "art", "circle", "join", "kick", "mask", + "master", "passion", "quick", "raise", "smooth", "unless", "wander", + "actually", "broke", "chair", "deal", "favorite", "gift", "note", "number", + "sweat", "box", "chill", "clothes", "lady", "mark", "park", "poor", + "sadness", "tie", "animal", "belong", "brush", "consume", "dawn", "forest", + "innocent", "pen", "pride", "stream", "thick", "clay", "complete", "count", + "draw", "faith", "press", "silver", "struggle", "surface", "taught", + "teach", "wet", "bless", "chase", "climb", "enter", "letter", "melt", + "metal", "movie", "stretch", "swing", "vision", "wife", "beside", "crash", + "forgot", "guide", "haunt", "joke", "knock", "plant", "pour", "prove", + "reveal", "steal", "stuff", "trip", "wood", "wrist", "bother", "bottom", + "crawl", "crowd", "fix", "forgive", "frown", "grace", "loose", "lucky", + "party", "release", "surely", "survive", "teacher", "gently", "grip", + "speed", "suicide", "travel", "treat", "vein", "written", "cage", "chain", + "conversation", "date", "enemy", "however", "interest", "million", "page", + "pink", "proud", "sway", "themselves", "winter", "church", "cruel", "cup", + "demon", "experience", "freedom", "pair", "pop", "purpose", "respect", + "shoot", "softly", "state", "strange", "bar", "birth", "curl", "dirt", + "excuse", "lord", "lovely", "monster", "order", "pack", "pants", "pool", + "scene", "seven", "shame", "slide", "ugly", "among", "blade", "blonde", + "closet", "creek", "deny", "drug", "eternity", "gain", "grade", "handle", + "key", "linger", "pale", "prepare", "swallow", "swim", "tremble", "wheel", + "won", "cast", "cigarette", "claim", "college", "direction", "dirty", + "gather", "ghost", "hundred", "loss", "lung", "orange", "present", "swear", + "swirl", "twice", "wild", "bitter", "blanket", "doctor", "everywhere", + "flash", "grown", "knowledge", "numb", "pressure", "radio", "repeat", + "ruin", "spend", "unknown", "buy", "clock", "devil", "early", "false", + "fantasy", "pound", "precious", "refuse", "sheet", "teeth", "welcome", + "add", "ahead", "block", "bury", "caress", "content", "depth", "despite", + "distant", "marry", "purple", "threw", "whenever", "bomb", "dull", "easily", + "grasp", "hospital", "innocence", "normal", "receive", "reply", "rhyme", + "shade", "someday", "sword", "toe", "visit", "asleep", "bought", "center", + "consider", "flat", "hero", "history", "ink", "insane", "muscle", "mystery", + "pocket", "reflection", "shove", "silently", "smart", "soldier", "spot", + "stress", "train", "type", "view", "whether", "bus", "energy", "explain", + "holy", "hunger", "inch", "magic", "mix", "noise", "nowhere", "prayer", + "presence", "shock", "snap", "spider", "study", "thunder", "trail", "admit", + "agree", "bag", "bang", "bound", "butterfly", "cute", "exactly", "explode", + "familiar", "fold", "further", "pierce", "reflect", "scent", "selfish", + "sharp", "sink", "spring", "stumble", "universe", "weep", "women", + "wonderful", "action", "ancient", "attempt", "avoid", "birthday", "branch", + "chocolate", "core", "depress", "drunk", "especially", "focus", "fruit", + "honest", "match", "palm", "perfectly", "pillow", "pity", "poison", "roar", + "shift", "slightly", "thump", "truck", "tune", "twenty", "unable", "wipe", + "wrote", "coat", "constant", "dinner", "drove", "egg", "eternal", "flight", + "flood", "frame", "freak", "gasp", "glad", "hollow", "motion", "peer", + "plastic", "root", "screen", "season", "sting", "strike", "team", "unlike", + "victim", "volume", "warn", "weird", "attack", "await", "awake", "built", + "charm", "crave", "despair", "fought", "grant", "grief", "horse", "limit", + "message", "ripple", "sanity", "scatter", "serve", "split", "string", + "trick", "annoy", "blur", "boat", "brave", "clearly", "cling", "connect", + "fist", "forth", "imagination", "iron", "jock", "judge", "lesson", "milk", + "misery", "nail", "naked", "ourselves", "poet", "possible", "princess", + "sail", "size", "snake", "society", "stroke", "torture", "toss", "trace", + "wise", "bloom", "bullet", "cell", "check", "cost", "darling", "during", + "footstep", "fragile", "hallway", "hardly", "horizon", "invisible", + "journey", "midnight", "mud", "nod", "pause", "relax", "shiver", "sudden", + "value", "youth", "abuse", "admire", "blink", "breast", "bruise", + "constantly", "couple", "creep", "curve", "difference", "dumb", "emptiness", + "gotta", "honor", "plain", "planet", "recall", "rub", "ship", "slam", + "soar", "somebody", "tightly", "weather", "adore", "approach", "bond", + "bread", "burst", "candle", "coffee", "cousin", "crime", "desert", + "flutter", "frozen", "grand", "heel", "hello", "language", "level", + "movement", "pleasure", "powerful", "random", "rhythm", "settle", "silly", + "slap", "sort", "spoken", "steel", "threaten", "tumble", "upset", "aside", + "awkward", "bee", "blank", "board", "button", "card", "carefully", + "complain", "crap", "deeply", "discover", "drag", "dread", "effort", + "entire", "fairy", "giant", "gotten", "greet", "illusion", "jeans", "leap", + "liquid", "march", "mend", "nervous", "nine", "replace", "rope", "spine", + "stole", "terror", "accident", "apple", "balance", "boom", "childhood", + "collect", "demand", "depression", "eventually", "faint", "glare", "goal", + "group", "honey", "kitchen", "laid", "limb", "machine", "mere", "mold", + "murder", "nerve", "painful", "poetry", "prince", "rabbit", "shelter", + "shore", "shower", "soothe", "stair", "steady", "sunlight", "tangle", + "tease", "treasure", "uncle", "begun", "bliss", "canvas", "cheer", "claw", + "clutch", "commit", "crimson", "crystal", "delight", "doll", "existence", + "express", "fog", "football", "gay", "goose", "guard", "hatred", + "illuminate", "mass", "math", "mourn", "rich", "rough", "skip", "stir", + "student", "style", "support", "thorn", "tough", "yard", "yearn", + "yesterday", "advice", "appreciate", "autumn", "bank", "beam", "bowl", + "capture", "carve", "collapse", "confusion", "creation", "dove", "feather", + "girlfriend", "glory", "government", "harsh", "hop", "inner", "loser", + "moonlight", "neighbor", "neither", "peach", "pig", "praise", "screw", + "shield", "shimmer", "sneak", "stab", "subject", "throughout", "thrown", + "tower", "twirl", "wow", "army", "arrive", "bathroom", "bump", "cease", + "cookie", "couch", "courage", "dim", "guilt", "howl", "hum", "husband", + "insult", "led", "lunch", "mock", "mostly", "natural", "nearly", "needle", + "nerd", "peaceful", "perfection", "pile", "price", "remove", "roam", + "sanctuary", "serious", "shiny", "shook", "sob", "stolen", "tap", "vain", + "void", "warrior", "wrinkle", "affection", "apologize", "blossom", "bounce", + "bridge", "cheap", "crumble", "decision", "descend", "desperately", "dig", + "dot", "flip", "frighten", "heartbeat", "huge", "lazy", "lick", "odd", + "opinion", "process", "puzzle", "quietly", "retreat", "score", "sentence", + "separate", "situation", "skill", "soak", "square", "stray", "taint", + "task", "tide", "underneath", "veil", "whistle", "anywhere", "bedroom", + "bid", "bloody", "burden", "careful", "compare", "concern", "curtain", + "decay", "defeat", "describe", "double", "dreamer", "driver", "dwell", + "evening", "flare", "flicker", "grandma", "guitar", "harm", "horrible", + "hungry", "indeed", "lace", "melody", "monkey", "nation", "object", + "obviously", "rainbow", "salt", "scratch", "shown", "shy", "stage", "stun", + "third", "tickle", "useless", "weakness", "worship", "worthless", + "afternoon", "beard", "boyfriend", "bubble", "busy", "certain", "chin", + "concrete", "desk", "diamond", "doom", "drawn", "due", "felicity", "freeze", + "frost", "garden", "glide", "harmony", "hopefully", "hunt", "jealous", + "lightning", "mama", "mercy", "peel", "physical", "position", "pulse", + "punch", "quit", "rant", "respond", "salty", "sane", "satisfy", "savior", + "sheep", "slept", "social", "sport", "tuck", "utter", "valley", "wolf", + "aim", "alas", "alter", "arrow", "awaken", "beaten", "belief", "brand", + "ceiling", "cheese", "clue", "confidence", "connection", "daily", + "disguise", "eager", "erase", "essence", "everytime", "expression", "fan", + "flag", "flirt", "foul", "fur", "giggle", "glorious", "ignorance", "law", + "lifeless", "measure", "mighty", "muse", "north", "opposite", "paradise", + "patience", "patient", "pencil", "petal", "plate", "ponder", "possibly", + "practice", "slice", "spell", "stock", "strife", "strip", "suffocate", + "suit", "tender", "tool", "trade", "velvet", "verse", "waist", "witch", + "aunt", "bench", "bold", "cap", "certainly", "click", "companion", + "creator", "dart", "delicate", "determine", "dish", "dragon", "drama", + "drum", "dude", "everybody", "feast", "forehead", "former", "fright", + "fully", "gas", "hook", "hurl", "invite", "juice", "manage", "moral", + "possess", "raw", "rebel", "royal", "scale", "scary", "several", "slight", + "stubborn", "swell", "talent", "tea", "terrible", "thread", "torment", + "trickle", "usually", "vast", "violence", "weave", "acid", "agony", + "ashamed", "awe", "belly", "blend", "blush", "character", "cheat", "common", + "company", "coward", "creak", "danger", "deadly", "defense", "define", + "depend", "desperate", "destination", "dew", "duck", "dusty", "embarrass", + "engine", "example", "explore", "foe", "freely", "frustrate", "generation", + "glove", "guilty", "health", "hurry", "idiot", "impossible", "inhale", + "jaw", + "kingdom", "mention", "mist", "moan", "mumble", "mutter", "observe", "ode", + "pathetic", "pattern", "pie", "prefer", "puff", "rape", "rare", "revenge", + "rude", "scrape", "spiral", "squeeze", "strain", "sunset", "suspend", + "sympathy", "thigh", "throne", "total", "unseen", "weapon", "weary" +] + +n = 1626 + + +# Note about US patent no 5892470: Here each word does not represent a given digit. +# Instead, the digit represented by a word is variable, it depends on the previous word. + + +def mn_encode(message): + assert len(message) % 8 == 0 + out = [] + for i in range(len(message) / 8): + word = message[8 * i:8 * i + 8] + x = int(word, 16) + w1 = (x % n) + w2 = ((x / n) + w1) % n + w3 = ((x / n / n) + w2) % n + out += [words[w1], words[w2], words[w3]] + return out + + +def mn_decode(wlist): + out = '' + for i in range(len(wlist) / 3): + word1, word2, word3 = wlist[3 * i:3 * i + 3] + w1 = words.index(word1) + w2 = (words.index(word2)) % n + w3 = (words.index(word3)) % n + x = w1 + n * ((w2 - w1) % n) + n * n * ((w3 - w2) % n) + out += '%08x' % x + return out + diff --git a/client/podle.py b/client/podle.py new file mode 100644 index 0000000..3a11d0e --- /dev/null +++ b/client/podle.py @@ -0,0 +1,659 @@ +#!/usr/bin/env python +from __future__ import print_function +#Proof Of Discrete Logarithm Equivalence +#For algorithm steps, see https://gist.github.com/AdamISZ/9cbba5e9408d23813ca8 +import os +import hashlib +import json +import binascii +PODLE_COMMIT_FILE = None +from btc import (multiply, add_pubkeys, getG, podle_PublicKey, podle_PrivateKey, + encode, decode, N, + podle_PublicKey_class, podle_PrivateKey_class) + + +def set_commitment_file(file_loc): + global PODLE_COMMIT_FILE + PODLE_COMMIT_FILE = file_loc + + +def get_commitment_file(): + return PODLE_COMMIT_FILE + + +class PoDLEError(Exception): + pass + + +class PoDLE(object): + """See the comment to PoDLE.generate_podle for the + mathematical structure. This class encapsulates the + input data, the commitment and the opening (the "proof"). + """ + + def __init__(self, + u=None, + priv=None, + P=None, + P2=None, + s=None, + e=None, + used=False): + #This class allows storing of utxo in format "txid:n" only for + #convenience of storage/access; it doesn't check or use the data. + #Arguments must be provided in hex. + self.u = u + if not priv: + if P: + #Construct a pubkey from raw hex + self.P = podle_PublicKey(binascii.unhexlify(P)) + else: + self.P = None + else: + if P: + raise PoDLEError("Pubkey should not be provided with privkey") + #any other formatting abnormality will just throw in PrivateKey + if len(priv) == 66 and priv[-2:] == '01': + priv = priv[:-2] + self.priv = podle_PrivateKey(binascii.unhexlify(priv)) + self.P = self.priv.pubkey + if P2: + self.P2 = podle_PublicKey(binascii.unhexlify(P2)) + else: + self.P2 = None + #These sig values should be passed in hex. + if s: + self.s = binascii.unhexlify(s) + if e: + self.e = binascii.unhexlify(e) + #Optionally maintain usage state (boolean) + self.used = used + #the H(P2) value + self.commitment = None + + def mark_used(self): + self.used = True + + def mark_unused(self): + self.used = False + + def get_commitment(self): + """Set the commitment to sha256(serialization of public key P2) + Return in hex to calling function + """ + if not self.P2: + raise PoDLEError("Cannot construct commitment, no P2 available") + if not isinstance(self.P2, podle_PublicKey_class): + raise PoDLEError("Cannot construct commitment, P2 is not a pubkey") + self.commitment = hashlib.sha256(self.P2.serialize()).digest() + return binascii.hexlify(self.commitment) + + def generate_podle(self, index=0, k=None): + """Given a raw private key, in hex format, + construct a commitment sha256(P2), which is + the hash of the value x*J, where x is the private + key as a raw scalar, and J is a NUMS alternative + basepoint on the Elliptic Curve; we use J(i) where i + is an index, so as to be able to create multiple + commitments against the same privkey. The procedure + for generating the J(i) value is shown in getNUMS(). + Also construct a signature (s,e) of Schnorr type, + which will serve as a zero knowledge proof that the + private key of P2 is the same as the private key of P (=x*G). + Signature is constructed as: + s = k + x*e + where k is a standard 32 byte nonce and: + e = sha256(k*G || k*J || P || P2) + + Possibly Joinmarket specific comment: + Users *should* generate with lower indices first, + since verifiers will give preference to lower indices + (each verifier may have their own policy about how high + an index to allow, which really means how many reuses of utxos + to allow in Joinmarket). + + Returns a commitment of form H(P2) which, note, will depend + on the index choice. Repeated calls will reset the commitment + and the associated signature data that can be used to open + the commitment. + """ + #TODO nonce could be rfc6979? + if not k: + k = os.urandom(32) + J = getNUMS(index) + KG = podle_PrivateKey(k).pubkey + KJ = multiply(k, J.serialize(), False, return_serialized=False) + self.P2 = getP2(self.priv, J) + self.get_commitment() + self.e = hashlib.sha256(''.join([x.serialize( + ) for x in [KG, KJ, self.P, self.P2]])).digest() + k_int = decode(k, 256) + priv_int = decode(self.priv.private_key, 256) + e_int = decode(self.e, 256) + sig_int = (k_int + priv_int * e_int) % N + self.s = encode(sig_int, 256, minlen=32) + return self.reveal() + + def reveal(self): + """Encapsulate all the data representing the proof + in a dict for client functions. Data output in hex. + """ + if not all([self.u, self.P, self.P2, self.s, self.e]): + raise PoDLEError("Cannot generate proof, data is missing") + if not self.commitment: + self.get_commitment() + Phex, P2hex, shex, ehex, commit = [ + binascii.hexlify(x) + for x in [self.P.serialize(), self.P2.serialize(), self.s, self.e, + self.commitment] + ] + return {'used': str(self.used), + 'utxo': self.u, + 'P': Phex, + 'P2': P2hex, + 'commit': commit, + 'sig': shex, + 'e': ehex} + + def serialize_revelation(self, separator='|'): + state_dict = self.reveal() + ser_list = [] + for k in ['utxo', 'P', 'P2', 'sig', 'e']: + ser_list += [state_dict[k]] + ser_string = separator.join(ser_list) + return ser_string + + @classmethod + def deserialize_revelation(cls, ser_rev, separator='|'): + ser_list = ser_rev.split(separator) + if len(ser_list) != 5: + raise PoDLEError("Failed to deserialize, wrong format") + utxo, P, P2, s, e = ser_list + return {'utxo': utxo, 'P': P, 'P2': P2, 'sig': s, 'e': e} + + def verify(self, commitment, index_range): + """For an object created without a private key, + check that the opened commitment verifies for at least + one NUMS point as defined by the range in index_range + """ + if not all([self.P, self.P2, self.s, self.e]): + raise PoDLE("Verify called without sufficient data") + if not self.get_commitment() == commitment: + return False + for J in [getNUMS(i) for i in index_range]: + sig_priv = podle_PrivateKey(self.s) + sG = sig_priv.pubkey + sJ = multiply(self.s, J.serialize(), False) + e_int = decode(self.e, 256) + minus_e = encode(-e_int % N, 256, minlen=32) + minus_e_P = multiply(minus_e, self.P.serialize(), False) + minus_e_P2 = multiply(minus_e, self.P2.serialize(), False) + KGser = add_pubkeys([sG.serialize(), minus_e_P], False) + KJser = add_pubkeys([sJ, minus_e_P2], False) + #check 2: e =?= H(K_G || K_J || P || P2) + e_check = hashlib.sha256(KGser + KJser + self.P.serialize() + + self.P2.serialize()).digest() + if e_check == self.e: + return True + #commitment fails for any NUMS in the provided range + return False + + +def getNUMS(index=0): + """Taking secp256k1's G as a seed, + either in compressed or uncompressed form, + append "index" as a byte, and append a second byte "counter" + try to create a new NUMS base point from the sha256 of that + bytestring. Loop counter and alternate compressed/uncompressed + until finding a valid curve point. The first such point is + considered as "the" NUMS base point alternative for this index value. + + The search process is of course deterministic/repeatable, so + it's fine to just store a list of all the correct values for + each index, but for transparency left in code for initialization + by any user. + + The NUMS generator generated is returned as a secp256k1.PublicKey. + """ + + assert index in range(256) + nums_point = None + for G in [getG(True), getG(False)]: + seed = G + chr(index) + for counter in range(256): + seed_c = seed + chr(counter) + hashed_seed = hashlib.sha256(seed_c).digest() + #Every x-coord on the curve has two y-values, encoded + #in compressed form with 02/03 parity byte. We just + #choose the former. + claimed_point = "\x02" + hashed_seed + try: + nums_point = podle_PublicKey(claimed_point) + return nums_point + except: + continue + assert False, "It seems inconceivable, doesn't it?" # pragma: no cover + + +def verify_all_NUMS(write=False): + """Check that the algorithm produces the expected NUMS + values; more a sanity check than anything since if the file + is modified, all of it could be; this function is mostly + for testing, but runs fast with pre-computed context so can + be run in user code too. + """ + nums_points = {} + for i in range(256): + nums_points[i] = binascii.hexlify(getNUMS(i).serialize()) + if write: + with open("nums_basepoints.txt", "wb") as f: + from pprint import pformat + f.write(pformat(nums_points)) + assert nums_points == precomp_NUMS, "Precomputed NUMS points are not valid!" + + +def getP2(priv, nums_pt): + """Given a secp256k1.PrivateKey priv and a + secp256k1.PublicKey nums_pt, an alternate + generator point (note: it's in no sense a + pubkey, its privkey is unknowable - that's + just the most easy way to manipulate it in the + library), calculate priv*nums_pt + """ + priv_raw = priv.private_key + return multiply(priv_raw, + nums_pt.serialize(), + False, + return_serialized=False) + + +def get_podle_commitments(): + """Returns set of commitments used as a list: + [H(P2),..] (hex) and a dict of all existing external commitments. + It is presumed that each H(P2) can + be used only once (this may not literally be true, but represents + good joinmarket "citizenship"). + This is stored as part of the data in PODLE_COMMIT_FILE + Since takers request transactions serially there should be no + locking requirement here. Multiple simultaneous taker bots + would require extra attention. + """ + if not os.path.isfile(PODLE_COMMIT_FILE): + return ([], {}) + with open(PODLE_COMMIT_FILE, "rb") as f: + c = json.loads(f.read()) + if 'used' not in c.keys() or 'external' not in c.keys(): + raise PoDLEError("Incorrectly formatted file: " + PODLE_COMMIT_FILE) + return (c['used'], c['external']) + + +def add_external_commitments(ecs): + """To allow external functions to add + PoDLE commitments that were calculated elsewhere; + the format of each entry in ecs must be: + {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}} + """ + update_commitments(external_to_add=ecs) + + +def update_commitments(commitment=None, + external_to_remove=None, + external_to_add=None): + """Optionally add the commitment commitment to the list of 'used', + and optionally remove the available external commitment + whose key value is the utxo in external_to_remove, + persist updated entries to disk. + """ + c = {} + if os.path.isfile(PODLE_COMMIT_FILE): + with open(PODLE_COMMIT_FILE, "rb") as f: + try: + c = json.loads(f.read()) + except ValueError: + print("the file: " + PODLE_COMMIT_FILE + " is not valid json.") + sys.exit(0) + + if 'used' in c: + commitments = c['used'] + else: + commitments = [] + if 'external' in c: + external = c['external'] + else: + external = {} + if commitment: + commitments.append(commitment) + #remove repeats + commitments = list(set(commitments)) + if external_to_remove: + external = { + k: v + for k, v in external.items() if k not in external_to_remove + } + if external_to_add: + external.update(external_to_add) + to_write = {} + to_write['used'] = commitments + to_write['external'] = external + with open(PODLE_COMMIT_FILE, "wb") as f: + f.write(json.dumps(to_write, indent=4)) + + +def generate_podle(priv_utxo_pairs, tries=1, allow_external=None, k=None): + """Given a list of privkeys, try to generate a + PoDLE which is not yet used more than tries times. + This effectively means satisfying two criteria: + (1) the generated commitment is not in the list of used + commitments + (2) the index required to generate is not greater than 'tries'. + Note that each retry means using a different generator + (see notes in PoDLE.generate_podle) + Once used, add the commitment to the list of used. + If we fail to find an unused commitment with this algorithm, + we fallback to sourcing an unused commitment from the "external" + section of the commitments file; if we succeed in finding an unused + one there, use it and add it to the list of used commitments. + If still nothing available, return None. + """ + used_commitments, external_commitments = get_podle_commitments() + for priv, utxo in priv_utxo_pairs: + for i in range(tries): + #Note that we will return the *lowest* index + #which is still available. + p = PoDLE(u=utxo, priv=priv) + c = p.generate_podle(i, k=k) + if c['commit'] in used_commitments: + continue + #persist for future checks + update_commitments(commitment=c['commit']) + return c + if allow_external: + filtered_external = dict([(x, external_commitments[x]) + for x in allow_external]) + for u, ec in filtered_external.iteritems(): + #use as many as were provided in the file, up to a max of tries + m = min([len(ec['reveal'].keys()), tries]) + for i in [str(x) for x in range(m)]: + p = PoDLE(u=u, + P=ec['P'], + P2=ec['reveal'][i]['P2'], + s=ec['reveal'][i]['s'], + e=ec['reveal'][i]['e']) + if p.get_commitment() not in used_commitments: + update_commitments(commitment=p.get_commitment()) + return p.reveal() + #If none of the entries in the 'reveal' list for this external + #commitment were available, they've all been used up, so + #remove this entry + if m == len(ec['reveal'].keys()): + update_commitments(external_to_remove=u) + #Failed to find any non-used valid commitment: + return None + + +def verify_podle(Pser, P2ser, sig, e, commitment, index_range=range(10)): + verifying_podle = PoDLE(P=Pser, P2=P2ser, s=sig, e=e) + #check 1: Hash(P2ser) =?= commitment + if not verifying_podle.verify(commitment, index_range): + return False + return True + + +precomp_NUMS = { + 0: '0296f47ec8e6d6a9c3379c2ce983a6752bcfa88d46f2a6ffe0dd12c9ae76d01a1f', + 1: '023f9976b86d3f1426638da600348d96dc1f1eb0bd5614cc50db9e9a067c0464a2', + 2: '023745b000f6db094a794d9ee08637d714393cd009f86087438ac3804e929bfe89', + 3: '023346660dcb1f8d56e44d23f93c3ad79761cdd5f4972a638e9e15517832f6a165', + 4: '02ec91c86964dcbb077c8193156f3cfa91476d5adfcfcf64913a4b082c75d5bca7', + 5: '02bbc5c4393395a38446e2bd4d638b7bfd864afb5ffaf4bed4caf797df0e657434', + 6: '02967efd39dc59e6f060bf3bd0080e8ecf4a22b9d1754924572b3e51ce2cde2096', + 7: '02cfce8a7f9b8a1735c4d827cd84e3f2a444de1d1f7ed419d23c88d72de341357f', + 8: '0206d6d6b1d88936bb6013ae835716f554d864954ea336e3e0141fefb2175b82f9', + 9: '021b739f21b981c2dcbaf9af4d89223a282939a92aee079e94a46c273759e5b42e', + 10: '025d72106845e03c3747f1416e539c5aa0712d858e7762807fdc4f3757fd980631', + 11: '02e7d4defb5d287734a0f96c2b390aa14f5f38e80c5a5e592e4ce10d55a5f5246b', + 12: '023c1bf301bcfa0f097f1a3931c68b4fd39b77a28cc7b61b2b1e0b7ca6d332493c', + 13: '0283ac2cdd6b362c90665802c264ee8e6342318070943717faee62ef9addeff3e9', + 14: '02cb9f6164cd2acdf071caef9deab870fc3d390a09b37ba7af8e91139b817ce807', + 15: '02f0a3a3e22c5b04b6fe97430d68f33861c3e9be412220dc2a24485ea5d55d94db', + 16: '02860ca3475757d90d999e6553e62c07fce5a6598d060cceeead08c8689b928095', + 17: '0246c8eabc38ce6a93868369d5900d84f36b2407eecb81286a25eb22684355b41d', + 18: '026aa6379d74e6cd6c721aef82a34341d1d15f0c96600566ad3fa8e9c43cbb5505', + 19: '02fdeacb3b4d15e0aae1a1d257b4861bcc9addb5dc3780a13eb982eb656f73d741', + 20: '021a83ecfaeb2c057f66a6b0d4a42bff3fe5fda11fe2eea9734f45f255444cddc0', + 21: '02d93580f3e0c2ec8ea461492415cc6a4be00c50969e2c32a2135e7d04f112309a', + 22: '0292c57be6c3e6ba8b44cf5e619529cf75e9c6b795ddecd383fb78f9059812cb3f', + 23: '02480f099771d0034d657f6b00cd17c7315b033b19bed9ca95897bc8189928dd47', + 24: '02ac0701cdc6f96c63752c01dc8400eab19431dfa15f85a7314b1e9a3df69a4a66', + 25: '026a304ceb69e37d655c1ef100d7ad23192867151983ab0d168af96afe7f1997f6', + 26: '023b9ff8e4a853b29ecae1e8312fae53863e86b8f8cb3155f31f7325ffb2baf02c', + 27: '021894ce66d61c33e439f38a36d92c0e45bf28dbc7e30bfb4d7135b87fc8e890e1', + 28: '02d9e7680e583cf904774d4c19f36cb3d238b6c770e1e7db03f444dc8b15b29687', + 29: '024350c7ff5b2bf2c58e3b17a792716d0e76cff7ad537375d1abc6e249466b25a3', + 30: '02c6577e1cdcbcfadb0ae037d01fbf6d74786eecdb9d1ee277d9ba69b969728cfe', + 31: '029f395b4c7b20bcb6120b57bee6d2f7353cd0aa9fe246176064068c1bd9b714d1', + 32: '02d180786087720b827bf04ae800547102470a1e43de314203e90228c586b481a1', + 33: '023548173a673965c18d994028bc6d5f5df1f60dccf9368b0eae34f8cff3106943', + 34: '02118124c53b86fdade932c4304ad347a19ce0af79a9ab885d7d3a6358a396e360', + 35: '02930bcdee5887fa5a258335d6948017e6d7f2665b32dcc76a84d5ca7cd604d89b', + 36: '0267e79a47058758a8ee240afd941e0ae8b4f175f29a3cf195ad6ff0e6d02955b1', + 37: '027e53d9fb04f1bb69324245306d26aa60172fd13d8fe27809b093222226914de6', + 38: '02ef09fbdcd22e1be4f0d4b2d13a141051b18009d7001f6828c6a40b145c9df23e', + 39: '028742fd08c60ba13e78913581db19af2f708c7ec53364589f6cbcf9d1c8b5105f', + 40: '020ce14308d2f516bf4f9e0944fb104907adef8f4c319bfcc3afab73e874a9ce4a', + 41: '027635f125f05a2548201f74c4bbdcbe89561204117bd8b82dfae29c85a576a58e', + 42: '02fe878f3ae59747ee8e9c34876b86851d5396124e1411f86fe5c58f08f413a549', + 43: '02f2a6af33bd08ab41a010d785694e9682fa1cc65733f30a53c40541d1c1bfb660', + 44: '02cbe9d18b6d5fc9993ef862892e5b2b1ea5d2710a4f208672c0f7c36a08bb5686', + 45: '023fb079b25c0a8241465fb55802f22ebb354e6da81f7dabfe214ddbd9d3dfcd5a', + 46: '021a5b234b9a10fc5f08ed9c1a136a250e92156adc12109a97dd7467276d6848a8', + 47: '0240fbe9363d50585da40aef95f311fc2795550e787f62421cd9b6e2f719bb9547', + 48: '02a245fbbc00f1d6feb72a9e1d3fd0033522839d33440aea64f52e8bccee616be8', + 49: '02fd1e94bb23a4306de64064841165e3db497ae5b246dabff738eb3e6ea51685a7', + 50: '0298362705914c839e45505369e54faefbb3aaebb4c486b4d6e59ca03304f3552c', + 51: '021b8109a23b858114d287273620dd920029d84b90f63af273c1c78492b1a70105', + 52: '028df6ce4fec30229cddb86c62606cff80e95cb8277028277f3dcc8ac9f98eef9d', + 53: '02ed02925d806df4ac764769d11743093708808157fb2933eb19af5399dcfd500c', + 54: '02ce88da0e81988bd8f5d63ad06898a355f7dc7f46bb08cf5f1e9bc5c3752ad13c', + 55: '02f4868cc8285cd8d74d4213d18d53d5f410d50223818f1be6fe8090904e03743d', + 56: '02770cecdf18aa2115b6e5c4295468f2e92a53068dc4295d0e5d0890b71d1a2fcc', + 57: '02b5d4dce8932de37c6ef13a7f063f164dfd07f7399e8e815e22b5af420608fd2a', + 58: '0284ad07924dbac50a72455aec3ddba50b1ed71e678ba935bb5d95c8a8232b1353', + 59: '02cb8c916a6f9bc39c8825f5b0378bb1b0a0679e191843aa4db2195b81f14c87e0', + 60: '0235aa30ec3df8dd193a132dbaf3b351af879c59504ed8b7b5ad5f1f1ea712854f', + 61: '02df91206e955cefe7bcda4555fc6ad761b0e98d464629f098d4483306851704e9', + 62: '02ed4f1fccd47e66a8d74e58b4f6e31b5172b628fc0dacdb408128c914eb80f506', + 63: '0263991bb62aaca78a128917f5c4e15183f98aefddf04070c5ca537186f1c1a97a', + 64: '02ffe2b017882d57db27864446ad7b21d3855ae64bddf74d46e3a611bf903580be', + 65: '02d647aba2c01eecd0fac7e82580dd8b92d66db7341d1b65a5e4b01234f1fbb2cd', + 66: '023134ff85401dba9aff426d3f3ba292ea59684b8c48ea0b495660797a839246a6', + 67: '02827880fe0410c9ea84f75a629f8f8e6eed1f26528af421cf23b8ecf93b6b4b7b', + 68: '02859b3f9f1f5ba6aa0787f8d3f3f2f21b4932c40bc36b6669383e3bbd19654a5f', + 69: '02a7d204dfc3eed44abd0419202e280f6772efd5acf9fd34331b8f47c81c6dab19', + 70: '02e15d11b443a9340ac31a8c5774ce34cd347834470c8d68c959828fae3a7eb0c6', + 71: '029931f65e46627d60519bfd08bd8a1bb3d8d2921f7f8c9ef31f4bfcdd8028ead2', + 72: '02e5415ba78743d736018f19757ee0e1ca5d4a4fb1d0464cd3eea8d89b34dd37b8', + 73: '027ea7860afc3de502d056d9a19ca330f16cd61cfefbeb768df68a882d1f8f15f5', + 74: '026c19becac43626582622e2b7e86ebd8056f40aa8ab031e70f4deae8cab34503f', + 75: '02098dab044c888ddebe6713fcb8481f178e3ba42d63310b08d8234e20fe1de13f', + 76: '02ed6af1a2bebcb381ce92f87638267b1afefe7a1cdce16253f5bf9f99a84ce4b2', + 77: '023d8493f9e72cd3212166de50940d980f603ae429309abb29e15cccc1983efe37', + 78: '025c07d7513b1bae52a0089a4faee127148e2ba5a651983083aedc1ae8403cf1eb', + 79: '0285a93a8c8e6134b3a53c5bd1b5b7d24e7911763ea887847c5d66af172ed17f10', + 80: '02fea28fb142aa95fcd44398c9482a3c185ec22fee8f24ad6b2297ac7478423f21', + 81: '02f9840a1635ae3fa131405526974d40d2edee17adf58278956373ce6c69757c2a', + 82: '023579e441a7dcdbd36a2932c64fa3318023b1f3d04daab148622b7646246a6d7c', + 83: '02bcbc2933f90a88996c1363c8d3a7004e0c6b75040041201fb022e45acb0af6a7', + 84: '02cd52e0d28f5564fc2bf842fa63dfefbcf2bb5fe0325703c132be5cd14cca7291', + 85: '021e648e261b93fedd3439352899c0fa1acedd1f68ab508050a13ed3cbbc93c2ff', + 86: '0295f9caea5f57d11b12ddee154a36a14921a8980fa76726e48e1d76443d4e306f', + 87: '02396edf4c18283dd3ef68a2c57b642bd87ae9f8b6be5e5fe4a41c5b86c5db8eb2', + 88: '0264f323ca3eee79385c9bfd35cd4cf576e51722f38dd98531d531a29913e5170d', + 89: '02facd3f63f543e0ab9b13323340113acbe8ed3bafdfabdc80626cdd15386c80f3', + 90: '02b6762640f96367fbf65eecfafcee5c6f7d6a42b706113053bb36a882659d3e65', + 91: '02ed63f2eca15d9b338fcdb9b3efa3b326e173a1390706258829680f7973fa851c', + 92: '026f6d47d0d48ff13d64ec6a1db2dc51173cee86ab8010a17809b3fe01483d9fc5', + 93: '02814e7cae580a1ef86d6ee9b2f9f26fe771e8ea47acf11153b04680ada9cd3042', + 94: '020e46225fb3ee8f04d08ffbe12d9092ff7f7227f9cb55709890c669e8a1c97963', + 95: '028194469e8d6ee660e95d6125ba0152ad5c24bf7e452adf80db7062d6926851c4', + 96: '02b3e1f5754562635ebeecfd32edb0d84a79b2f0c270bac153e60dd29334dc2663', + 97: '02afff20730724a2d422f330e962362e7831753545ac0a931dd94be011ccf93e9c', + 98: '02a9cfdf0471a34babfc2f6201dbc79530f3f319204daedb7ec05effc2bdfc5a74', + 99: '02838fe450f2dd0c460b5fae90ec2feb5b7f001f9cd14c01a475c492cf16ea594b', + 100: '02aacc3145d04972d0527c4458629d328219feda92bef6ef6025878e3a252e105a', + 101: '02720fe09616d4325d3c4c702a0aeafbbbff95ef962af531c5ae9461ec81fdf8c5', + 102: '02e6408f24461a6c484f6c4493c992d303211d5e4297d34afede719a2b70c96c14', + 103: '02b9ecf2d3fdf2611c6d4be441a0f9a3810dadae39feb3c0d855748cc2dd98a968', + 104: '027a32d12a536af038631890a9b90ee20b219c9c8231a95b1cde24c143d8173fec', + 105: '02d26c98fb50b57b7defdf1e8062a52b2a859ba42f3d1760ee8ff99c4e9eb3ec03', + 106: '02df85556e8d1e97a8093e4d9950905ebced0ea9a1e49728713df1974eeb455774', + 107: '021fe1dbada397155a80225b59b4fb9a32450a991b2d9d11d8500e98344927c856', + 108: '0211ccd0980a9ab6f4bb82fdc2e2d1ddace063a7bc1914a6ab4d02b0fa1ca746ec', + 109: '0264bd41f41aad19f8bfd290fd3af346ebbf80efd33f515854f82bd57e9740f7aa', + 110: '0226d5fb607cadb8720e900ce9afb9607386ad7b767e4ab3a4e0966223324b92eb', + 111: '02b3bbf2e2ceae25701bd3b78ba13bea3f0dfed7581b8a8a67c66de9fd96ee41e2', + 112: '024b8dd765e385d0e04772f3dbf1b1a82abc2de3e5740baac1f6306cd9fd45fe99', + 113: '022153f6a884ae893ebb0642a84d624c0b62894d7cb9e2a48a3a0c4696e593f9db', + 114: '0245e22b6388cb14c9c8dbcac94853bdf1e81816c07e926a82b96fc958aa874626', + 115: '02cba97826b089c695b1acffdcdbf1484beec5eb95853fea1535d6d7bdb4e678b0', + 116: '02ed006fbab2d18adbd96d2f1de6b83948e2a47acc8d2f92d7af9ba01ffae58276', + 117: '02513592f4434ee62802d3965f847684693000830107c72cd8de4b34e05b532dae', + 118: '028adc75647453a247bd44855abb56b60794aaed5ce21c9898e62adac7adcfbe8e', + 119: '02a712d5dc572086359f1688e8e7b9a5f7fc3079644aea27cdddb382208fee885b', + 120: '029abf8551218c9076f6d344baa099041fe73e5e844aac6e5c20240834105cdf60', + 121: '027d480071a2d128c51e84c380467e1ac8435f05b985bbfee0099d35b4121fb0ca', + 122: '02a7f2e4253fa0d833beca742e210c0d59a4ffc8559764766dcffb1aa3e4961826', + 123: '023521309a6bdfafdf7bdae574a5f6010eb992e4bae46d8f83c478eac137889270', + 124: '02b99fe8623aa19ca2bed6fe435ae95c5072a40193913bebe5466f675c92a31db7', + 125: '02dc035112a2b4881917ea1db159e7f35ee9d98d31533e1285ca150ce84e538e4f', + 126: '0291a07ecce8061561624de7348135b9081c5edd61541b24fa002fb6c074318fec', + 127: '020d8a5253d7e0166aa37680a5f64cab0cdad2cdc4c0e8ae61d310df4c4f7386eb', + 128: '026285db47fee60b5ad54cbd4c27a4e0cd723b86a920f03b12dc9b8c5f19f06448', + 129: '020f94a9df4302f701b4629f74d401484daf84c7aabaf533f8c21c1626009e923c', + 130: '027bb78af54b01ddad4e96b51a4e024105b373aab7e1a6ec16279967fcbbb096b4', + 131: '02e1b20c0da3b8c991f8909fd0d31874be00e9fcb130d7c28b8ad53326cdf13755', + 132: '02bbdd4dfc047f216e2cbff789bcf850423bedf2006d959963f75621810fecf0d9', + 133: '024e1fe4b23feda8651a467090e0ce7e8b8db2ccb1c27d52255c76754aa1940d1b', + 134: '0241aad8f575556c49c4fefae178c2c38541962bfff2ca84ebecea9f661ccf3536', + 135: '02bcf6203d725ca0640bd045389e854e00087c54ba01fd739c6ef685b22f89340c', + 136: '0202178e6b3a9b498399aa392b32dc9010f1eea322a6d439ad0c8cacf2008b3e34', + 137: '026db3289d470df0fdf04f5f608fae2d7ec4ddbd3de2603f6685789520bdee01fc', + 138: '0239bcfc796488129e3b2f01e6fbbda2f1b357b602e94b5091b44c916e9806dc34', + 139: '020513bc4a618d32d784083f13d46e6c6d547f01b24942351760f6dc42e2bb7167', + 140: '0204d2495e4fc20e0571ab2fcb4c1989fdda4542923aa97fe1a77a11c79ade1964', + 141: '021eaa6af99ea4f1143a45a1b5af7b2d3c3e8810f358be6261248c5ba2492a7b4e', + 142: '02799849e87e3862170add5b28a3b7e54b04cc60c2cec39de7eca9bfdfaaf930a8', + 143: '02639bced287084268136c5b6e9e22f743b6c8f813e6aabe39521715bfa4a46ab8', + 144: '0283c8b21fc038c1fbeedfae0b3abc4dbde672b0dcfda540f9fcfcf8c6e6d29fc3', + 145: '02b284f4510535ff98e683f25c08b7ae7dd19f7b861e70a202469ddfb2877bc729', + 146: '0256af1c82cde40ffd03564368b8256a5e48ef056df2655013f0b1aa15de1de8d2', + 147: '02964b55eab2f19518ee735cae2f7f780bfab480bcbd360f7a90a2904301203366', + 148: '02f046486f4a473f2226f6bd120aafc55a5c8651f3eb0855aa6a821f69f3016cc6', + 149: '02eb8dfb7c59fbf24671e258ca5e8eda3ea74c5f0455eed4987cfda79f4fcf823f', + 150: '020fac2c37cc273d982c07b2719a3694348629d5bdaebc22967fb9d0e1d7f01842', + 151: '025c0c8ff9a102f99f700081526d2b93b9d51caf81dcf4d02e93cf83b4a7ff5c92', + 152: '02a118f5fa9c5ef02707e021f9cb8056e69018ef145bec80ead4e09c06a60050c1', + 153: '029ea72333d1908bb082bffec9da8824883df76a89709ab090df86c45be4abf784', + 154: '02bacc52256e5221dbfc9a3f22e30fa8e86ddd38e3877e3dc41de91bdcf989b00b', + 155: '02bc8b37dc66e2296ae706c896f5b86bd335f724cfa9783e41b9dc5e901b42b1de', + 156: '02eca1099cea9bcab80820d6b64aec16dce1efa0e997b589f6dba3a8fd391fb100', + 157: '027f1c1bb99bd1a0e486f415f8960d45614a6fcac8cedc260e07197733844827d0', + 158: '021fc54df458bcfafc8a83d4759224c49c4b338cf23cd9825d6e9cdeffc276375b', + 159: '027d4fff88da831999ba9b21e19baf747dc26ea76651146e463d4f3e51c586ee91', + 160: '02e49c0fef0ebc52908cdcea4d913a42e5f24439fffdfaa21cc55a6add0ad9d122', + 161: '0208b5e8e5035fdb62517d4ebab0696775dbfbdba8ff80f2031c1156cda195a2ab', + 162: '0202e990bab267fff1575d6acc76fe2041f4196f4b17678872f9c160d930e5be35', + 163: '02c73fcedd9f6eabc8fe4e1e7211cdb0f28967391d200147d46e4077d2915c262d', + 164: '0261490abc5f14387ef585f42d99dbddb0837b166694d4af521086a1ffd46e5640', + 165: '02b46a143e4e0af20a12c39f3105aca57ca79f9332df67619ee859b5d9bffb6d6d', + 166: '0299f53c064d068f003f8871acae31b84ddda9d8dbe516d02dc170c70314ee2af7', + 167: '023305144dccba65c67001474ee1135aa96432f386b5eb27582393b2ed4bfc185d', + 168: '02e044b70ff7e9c784b3c40d09bdfadd4a037e692b0b3aa9ab6bb91203f86a0b37', + 169: '02ded067a2e44282b0d731a28ffbd03ca6046c5b1a262887ea7cab4810050fbb8c', + 170: '02e00e4c9198194d92a93059bce61f8249e1006eee287aa94fe51bb207462e5492', + 171: '0241b89d9164f4c07595ca99b7d73cad2b20ac39847cf703dff1d7d6add339ebeb', + 172: '02eba24cd4946e149025a9bf7759df5362245bf7c53c5a3205be0c92c59db8d5dc', + 173: '026bd40c611246a789521c46d758a80337ff40bb298a964612b2af74039211727a', + 174: '02b9095e071e4edfddf8afb0e176536957509d23f90fb7175ad086b4098e731c73', + 175: '0214ad0014dfddc5c7eb0801b97268c1b7e03d64215d6b9d5ed80b468089e4a01d', + 176: '02c455b8e38103ade8794fb51a1656e1439b42bdf79afd17a9df8542153914a7cf', + 177: '02cc89d6437fdcf711a76eb16f4014f2e21b71740afc8b3ec13ccb60a45b12d815', + 178: '0208eee5857dda0ae1c721e6ed4c74044add4e1ce66f105413e9ef1cccbdca87ad', + 179: '02edc663693827cad44d004ac24753bfc3167f81ff4074bb862453376593229c0f', + 180: '0202a4b7fb31e30b6d8f90a5442ef31f800902ea7a9511e24437b7a0ef516f79a9', + 181: '02ff05472c2019ac2c9ab8b7fcb0604a94b7379c350306be262144588ea252d0f4', + 182: '02b131bb594a1270d231e18459e484c49f3eca3b3b2291c9be81c01dc8a4037fa1', + 183: '02f50125277ea19f633e93868cf8e8a4cd76b21eedf8e3ef59de43f40d73a01d01', + 184: '027aab228a7d6f87003b01fb9c0b9bcfb2098adbc76f5f9b856aedd28077fc4471', + 185: '02925200e4f74bea719a99f4a0b05165b9af475f2187381bd0b79cad4d5f2593b6', + 186: '02c311f1750c6d5c364b71c3b0f369f6959d34a3718da695c5b227ecf1a4669bf6', + 187: '02cb030c71169d0a1ae30ffba92311bc06bb64b27570598dedabdea0b24631a0ca', + 188: '02e64669898eecff7aa887307be696a694f61559e7ca41119677b7e94f37cd2914', + 189: '028fe93e32c24df7f8aaf8d777335fd9ce9f9b5c121dec2ab1ff21575c047497e7', + 190: '026f08c1c3cb4cff5cdbd7985db4a8ebf0ebc0924530b0fa118d095c4667efeb52', + 191: '02afe08dbba6c999efb73aeae1da0ad8b143a1b51759caffd3ed2de4494adc47fb', + 192: '02e99aec0b5e869b3885a3b9f527fd3c546dde83d41a5a156703d0da5e10e04743', + 193: '02b7e5f4cb9233107bf7a47789dca4eb811af108822f2d4bd03dec13251ec45984', + 194: '023b971e135daa0b851797b17e3a1cc5ac8a9a6207a2e784a0fe36732a00407b49', + 195: '02b1742739bfbb528b2a2731cb5d5f1bd03f4fa9c94607837e586c7c6f6589be4a', + 196: '022cd1b023bb2afc68ee27b40f8deb1d1c6d7b7aa97c32c444f1ceebd449dbeb22', + 197: '02704e21f8bf38158d7e8100e297adfc930c14c8791beee9b907407f4ca654d95b', + 198: '02caabeb678374ca75bd815c370b2e37fb0470591557219d6289b1b1e655ed80c6', + 199: '026aa8d45112aa0da335054194c739e04787526250493f5a0eaaa8a346541d1a0f', + 200: '022fb12408355439bbee33066bbeefcffb0bdc9cfd1950510fd2a42bdc4eaa1d53', + 201: '02639fe47769f7694ca6dbfd934762472391d70b23868a58e11d2bd46373e1df29', + 202: '02f75360f52df674247c5f005b3451ee47becf3204862154d4e7ee97a0e40df3d2', + 203: '0230241e27d0d3ad727d26472541fcd48f2bb128db5611237fa9f33f86ede8d5c9', + 204: '0255d5a0aa37a226c001f6b7f19e2bddb10aeaa0652430b8defe35c3f03dfb3c0e', + 205: '024e6faa398b0acf8a8dfdd9d21e0a46a22d07cd0fcffd89749f74f94f9993f4d9', + 206: '020c1a256587306f58f274cc2238f651bbfadfd42436e6eb8f318ac08fae04e7ae', + 207: '025858b8188da173e8b01b8713b154ffae8b2d2eb8f9670362877102cf0c0c4f28', + 208: '02dc7509c77d7fa61c08c5525fb151bf4fe12deb1989a3be560a63105dae2ecd2e', + 209: '02a272df6dab1c22c209b45b601737c0077acb7869bb9fe264c991b4ef199e337d', + 210: '025168f2fdd730b4c33b57d3956e6a40dd27a4f32db70d9f9b5898fa2bed3de342', + 211: '028133baac70bc2c2ebe8a22af04b5faedd070e276c90e2f910bb9bf89441a80db', + 212: '029064628ebd6e97a945c1d52641a27bff3c4f59659e657b88d23c2ce1c4d04644', + 213: '023cf20c4e8675bce999a0128602fe21699db651540f3dcbe7a4ef2126243ba17a', + 214: '02cc685739a4b20e2d52ddf256e597c06b7eb69e65d009820c6744b739c7215340', + 215: '02d061544ce21398af3e0e6c329ce49976a9ecd804ebc543f4c16f6a32798f37c2', + 216: '029fe49ff440f23c69360a92d249db429bdc3601fc8a5a3fc1aa894de817c05490', + 217: '0222c8c4e90585f9816b5801bad43fb608857269fdaaefbe2b5b85903231685679', + 218: '0296b72ed4968860b733fb99846698df2e95c65af281b3ef8b5ab90e2d5de966cb', + 219: '02c27565a7fd5d1f4bcbe969bddbace99553fb65cb7750965350ff230b1f09f97d', + 220: '02e1254be9833236609bf44c62ef6da7188a44bbe2d53a72cf39a38ef9f99bb783', + 221: '0280663ce16afadc77e00ade780da53e7c11b02a66cbf36837ef7d9d2488f23417', + 222: '02ad8b11e62c6753917307bdde89a42896e0070d33f6f93c608d82f6d041b814a4', + 223: '02ce1d943dfc14654266507def2b7b9940bffceb4f54d709a149f99962083398fc', + 224: '023ea7eb26248c05beb4e4d8ba9f9785d5fd1a55d3137c90f40b807b60aa4262df', + 225: '0211c802fec9b31710d3849e2c1700cea5374ae422e54551946d96fc240c63fba0', + 226: '02204ad97ebe2ec30d6db1bfc1e1d4660331909668634c3cd928b5c369a6013367', + 227: '020251bf4271d359a082cdad23d9a5cd48916d78eed010fe1e7d9711cd420b3cdf', + 228: '0292b9757195350676e447e49425f887d3df7e27774bb3e0aab5b528da0a1a0340', + 229: '022be18362b2a167199a76f6065358063b1167d5bbcfe7652fc55f93a5ebd42e89', + 230: '02e6b1e618efe5f468bdb40f5ec167ed4fa7636849c4ff4ddab0199c903b37306c', + 231: '02a6676873de91890ecae000c575e46e4a9629865fb1662606da5e9c1fdcd55d5c', + 232: '02c088a3c96b13413caa5f32a8f4640e76ec0a37990577d679d2062e859547f058', + 233: '023e9703ed6209d5a25e0ecb34e04c22f274f37845aa2a4e2f2343e39928360e25', + 234: '02977d845787c4690152827bfd15e801044c84d33430a7ed928499e828cf131d14', + 235: '0224ea648555445d1305aaf6bd74fda3041b2a10bf7900a4c067462b01c6dc25f1', + 236: '02dfd472c98ece1dc2a18c1bebf98a09990fba673e725c029928937247022b9d24', + 237: '02a2a03933d06617adcf0f4ad692e95d463a5fa9938e8d451e5d6271f4a5af8bb4', + 238: '02ca24fa8d7aa53f7f5b4e1ca16eb6fd9b9cfb0162a332abb7a88ddf8e964c99bc', + 239: '02bbce92d1db3ef0c9c09793b760fd3b929c9168e4dff396c618fa0ed3cf6a5edb', + 240: '028af15d26d3b297f4d2aeaf308632b60251accf87aa8470b3d4d1ef2dabb99209', + 241: '021b81c0e878389231339fd9d622a736fc9d36de93a58ea6a4bc38fef86672278a', + 242: '021adc24309f605c7a5af106e8b930feaec0bec6545fb4c70b83ebe5cf341cab2d', + 243: '020462a3ff101ac379f87f43190459b7494f4128ea30035877ce22a35afb995e34', + 244: '02f1019851779a6d0db09e8abeba3b9a07b6931b43b0d973cfe261a96b4516cca4', + 245: '02d7023276f01ff22a9efeadd5b539d1d9ceb80ebf6813e6042a49c946a82f366f', + 246: '021594f45af3a21e0210a2ca4cbc3e95ea95db5aca3561fc1f759cb7f104dd0f62', + 247: '021398309b6c293c0dc28cdd7e55ad06306b59cb9c10d947df565e4a90f095a62a', + 248: '029f39d84383200e841187c5b0564e3b01a2ba019b86221c0c1dd3eae1b4dabb26', + 249: '0252ec719852f71c2d58886dd6ace6461a64677a368b7b8e220da005ac977abdc8', + 250: '0237f0d7de84b2cc6d2109b7241c3d49479066a09d1412c7a4734192715b021e06', + 251: '021e9e0e4784d15a29721c9a33fbcfb0af305d559c98a38dcf0ce647edd2c50caa', + 252: '02e705994a78f7942726209947d62d64edd062acfa8a708c21ac65de71e7ae71df', + 253: '0295f1cafd97e026341af3670ef750de4c44c82e6882f65908ec167d93d7056806', + 254: '023a0d381598e185bbff88494dc54e0a083d3b9ce9c8c4b86b5a4c9d5f949b1828', + 255: '02a0a8694820c794852110e5939a2c03f8482f81ed57396042c6b34557f6eb430a' +} diff --git a/client/slowaes.py b/client/slowaes.py new file mode 100644 index 0000000..b516094 --- /dev/null +++ b/client/slowaes.py @@ -0,0 +1,674 @@ +#!/usr/bin/python +# +# aes.py: implements AES - Advanced Encryption Standard +# from the SlowAES project, http://code.google.com/p/slowaes/ +# +# Copyright (c) 2008 Josh Davis ( http://www.josh-davis.org ), +# Alex Martelli ( http://www.aleax.it ) +# +# Ported from C code written by Laurent Haan ( http://www.progressive-coding.com ) +# +# Licensed under the Apache License, Version 2.0 +# http://www.apache.org/licenses/ +# +import math +import os + + +def append_PKCS7_padding(s): + """return s padded to a multiple of 16-bytes by PKCS7 padding""" + numpads = 16 - (len(s) % 16) + return s + numpads * chr(numpads) + + +def strip_PKCS7_padding(s): + """return s stripped of PKCS7 padding""" + if len(s) % 16 or not s: + raise ValueError("String of len %d can't be PCKS7-padded" % len(s)) + numpads = ord(s[-1]) + if numpads > 16: + raise ValueError("String ending with %r can't be PCKS7-padded" % s[-1]) + if not all(numpads == x for x in map(ord, s[-numpads:-1])): + raise ValueError("Invalid PKCS7 padding") + return s[:-numpads] + + +class AES(object): + # valid key sizes + keySize = dict(SIZE_128=16, SIZE_192=24, SIZE_256=32) + + # Rijndael S-box + sbox = [0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, + 0x2b, 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, + 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, + 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, + 0x71, 0xd8, 0x31, 0x15, 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, + 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, + 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, + 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, + 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 0xd0, 0xef, 0xaa, + 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, + 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, + 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec, + 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, + 0x73, 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, + 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, + 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, + 0xea, 0x65, 0x7a, 0xae, 0x08, 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, + 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 0x70, + 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, + 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, + 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 0x8c, 0xa1, + 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, + 0x54, 0xbb, 0x16] + + # Rijndael Inverted S-box + rsbox = [0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, + 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, + 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 0x54, + 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, + 0x42, 0xfa, 0xc3, 0x4e, 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, + 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 0x72, 0xf8, + 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, + 0x65, 0xb6, 0x92, 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, + 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, 0x90, 0xd8, 0xab, + 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, + 0x45, 0x06, 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, + 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 0x3a, 0x91, 0x11, 0x41, + 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, + 0x73, 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, + 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 0x47, 0xf1, 0x1a, 0x71, 0x1d, + 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, + 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, + 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 0x60, + 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, + 0x93, 0xc9, 0x9c, 0xef, 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, + 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 0x17, 0x2b, + 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, + 0x21, 0x0c, 0x7d] + + def getSBoxValue(self, num): + """Retrieves a given S-Box Value""" + return self.sbox[num] + + def getSBoxInvert(self, num): + """Retrieves a given Inverted S-Box Value""" + return self.rsbox[num] + + @staticmethod + def rotate(word): + """ Rijndael's key schedule rotate operation. + + Rotate a word eight bits to the left: eg, rotate(1d2c3a4f) == 2c3a4f1d + Word is an char list of size 4 (32 bits overall). + """ + return word[1:] + word[:1] + + # Rijndael Rcon + Rcon = [0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, + 0x6c, 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, + 0x35, 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, + 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, + 0xcc, 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, + 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, + 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, + 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, + 0xc2, 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, + 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, + 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, + 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, + 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f, 0x25, 0x4a, + 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb, 0x8d, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, + 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, + 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, + 0xd3, 0xbd, 0x61, 0xc2, 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, + 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, 0x08, + 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, + 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, 0x7d, + 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, + 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, 0x74, + 0xe8, 0xcb] + + def getRconValue(self, num): + """Retrieves a given Rcon Value""" + return self.Rcon[num] + + def core(self, word, iteration): + """Key schedule core.""" + # rotate the 32-bit word 8 bits to the left + word = self.rotate(word) + # apply S-Box substitution on all 4 parts of the 32-bit word + for i in range(4): + word[i] = self.getSBoxValue(word[i]) + # XOR the output of the rcon operation with i to the first part + # (leftmost) only + word[0] = word[0] ^ self.getRconValue(iteration) + return word + + def expandKey(self, key, size, expandedKeySize): + """Rijndael's key expansion. + + Expands an 128,192,256 key into an 176,208,240 bytes key + + expandedKey is a char list of large enough size, + key is the non-expanded key. + """ + # current expanded keySize, in bytes + currentSize = 0 + rconIteration = 1 + expandedKey = [0] * expandedKeySize + + # set the 16, 24, 32 bytes of the expanded key to the input key + for j in range(size): + expandedKey[j] = key[j] + currentSize += size + + while currentSize < expandedKeySize: + # assign the previous 4 bytes to the temporary value t + t = expandedKey[currentSize - 4:currentSize] + + # every 16,24,32 bytes we apply the core schedule to t + # and increment rconIteration afterwards + if currentSize % size == 0: + t = self.core(t, rconIteration) + rconIteration += 1 + # For 256-bit keys, we add an extra sbox to the calculation + if size == self.keySize["SIZE_256"] and ( + (currentSize % size) == 16): + for l in range(4): + t[l] = self.getSBoxValue(t[l]) + + # We XOR t with the four-byte block 16,24,32 bytes before the new + # expanded key. This becomes the next four bytes in the expanded + # key. + for m in range(4): + expandedKey[currentSize] = expandedKey[currentSize - size] ^ \ + t[m] + currentSize += 1 + + return expandedKey + + @staticmethod + def addRoundKey(state, roundKey): + """Adds (XORs) the round key to the state.""" + for i in range(16): + state[i] ^= roundKey[i] + return state + + @staticmethod + def createRoundKey(expandedKey, roundKeyPointer): + """Create a round key. + Creates a round key from the given expanded key and the + position within the expanded key. + """ + roundKey = [0] * 16 + for i in range(4): + for j in range(4): + roundKey[j * 4 + i] = expandedKey[roundKeyPointer + i * 4 + j] + return roundKey + + @staticmethod + def galois_multiplication(a, b): + """Galois multiplication of 8 bit characters a and b.""" + p = 0 + for counter in range(8): + if b & 1: p ^= a + hi_bit_set = a & 0x80 + a <<= 1 + # keep a 8 bit + a &= 0xFF + if hi_bit_set: + a ^= 0x1b + b >>= 1 + return p + + # + # substitute all the values from the state with the value in the SBox + # using the state value as index for the SBox + # + def subBytes(self, state, isInv): + if isInv: + getter = self.getSBoxInvert + else: + getter = self.getSBoxValue + for i in range(16): + state[i] = getter(state[i]) + return state + + # iterate over the 4 rows and call shiftRow() with that row + def shiftRows(self, state, isInv): + for i in range(4): + state = self.shiftRow(state, i * 4, i, isInv) + return state + + # each iteration shifts the row to the left by 1 + @staticmethod + def shiftRow(state, statePointer, nbr, isInv): + for i in range(nbr): + if isInv: + state[statePointer:statePointer + 4] = \ + state[statePointer + 3:statePointer + 4] + \ + state[statePointer:statePointer + 3] + else: + state[statePointer:statePointer + 4] = \ + state[statePointer + 1:statePointer + 4] + \ + state[statePointer:statePointer + 1] + return state + + # galois multiplication of the 4x4 matrix + def mixColumns(self, state, isInv): + # iterate over the 4 columns + for i in range(4): + # construct one column by slicing over the 4 rows + column = state[i:i + 16:4] + # apply the mixColumn on one column + column = self.mixColumn(column, isInv) + # put the values back into the state + state[i:i + 16:4] = column + + return state + + # galois multiplication of 1 column of the 4x4 matrix + def mixColumn(self, column, isInv): + if isInv: + mult = [14, 9, 13, 11] + else: + mult = [2, 1, 1, 3] + cpy = list(column) + g = self.galois_multiplication + + column[0] = g(cpy[0], mult[0]) ^ g(cpy[3], mult[1]) ^ \ + g(cpy[2], mult[2]) ^ g(cpy[1], mult[3]) + column[1] = g(cpy[1], mult[0]) ^ g(cpy[0], mult[1]) ^ \ + g(cpy[3], mult[2]) ^ g(cpy[2], mult[3]) + column[2] = g(cpy[2], mult[0]) ^ g(cpy[1], mult[1]) ^ \ + g(cpy[0], mult[2]) ^ g(cpy[3], mult[3]) + column[3] = g(cpy[3], mult[0]) ^ g(cpy[2], mult[1]) ^ \ + g(cpy[1], mult[2]) ^ g(cpy[0], mult[3]) + return column + + # applies the 4 operations of the forward round in sequence + def aes_round(self, state, roundKey): + state = self.subBytes(state, False) + state = self.shiftRows(state, False) + state = self.mixColumns(state, False) + state = self.addRoundKey(state, roundKey) + return state + + # applies the 4 operations of the inverse round in sequence + def aes_invRound(self, state, roundKey): + state = self.shiftRows(state, True) + state = self.subBytes(state, True) + state = self.addRoundKey(state, roundKey) + state = self.mixColumns(state, True) + return state + + # Perform the initial operations, the standard round, and the final + # operations of the forward aes, creating a round key for each round + def aes_main(self, state, expandedKey, nbrRounds): + state = self.addRoundKey(state, self.createRoundKey(expandedKey, 0)) + i = 1 + while i < nbrRounds: + state = self.aes_round(state, + self.createRoundKey(expandedKey, 16 * i)) + i += 1 + state = self.subBytes(state, False) + state = self.shiftRows(state, False) + state = self.addRoundKey( + state, self.createRoundKey(expandedKey, 16 * nbrRounds)) + return state + + # Perform the initial operations, the standard round, and the final + # operations of the inverse aes, creating a round key for each round + def aes_invMain(self, state, expandedKey, nbrRounds): + state = self.addRoundKey( + state, self.createRoundKey(expandedKey, 16 * nbrRounds)) + i = nbrRounds - 1 + while i > 0: + state = self.aes_invRound(state, + self.createRoundKey(expandedKey, 16 * i)) + i -= 1 + state = self.shiftRows(state, True) + state = self.subBytes(state, True) + state = self.addRoundKey(state, self.createRoundKey(expandedKey, 0)) + return state + + # encrypts a 128 bit input block against the given key of size specified + def encrypt(self, iput, key, size): + output = [0] * 16 + # the number of rounds + nbrRounds = 0 + # the 128 bit block to encode + block = [0] * 16 + # set the number of rounds + if size == self.keySize["SIZE_128"]: + nbrRounds = 10 + elif size == self.keySize["SIZE_192"]: + nbrRounds = 12 + elif size == self.keySize["SIZE_256"]: + nbrRounds = 14 + else: + return None + + # the expanded keySize + expandedKeySize = 16 * (nbrRounds + 1) + + # Set the block values, for the block: + # a0,0 a0,1 a0,2 a0,3 + # a1,0 a1,1 a1,2 a1,3 + # a2,0 a2,1 a2,2 a2,3 + # a3,0 a3,1 a3,2 a3,3 + # the mapping order is a0,0 a1,0 a2,0 a3,0 a0,1 a1,1 ... a2,3 a3,3 + # + # iterate over the columns + for i in range(4): + # iterate over the rows + for j in range(4): + block[(i + (j * 4))] = iput[(i * 4) + j] + + # expand the key into an 176, 208, 240 bytes key + # the expanded key + expandedKey = self.expandKey(key, size, expandedKeySize) + + # encrypt the block using the expandedKey + block = self.aes_main(block, expandedKey, nbrRounds) + + # unmap the block again into the output + for k in range(4): + # iterate over the rows + for l in range(4): + output[(k * 4) + l] = block[(k + (l * 4))] + return output + + # decrypts a 128 bit input block against the given key of size specified + def decrypt(self, iput, key, size): + output = [0] * 16 + # the number of rounds + nbrRounds = 0 + # the 128 bit block to decode + block = [0] * 16 + # set the number of rounds + if size == self.keySize["SIZE_128"]: + nbrRounds = 10 + elif size == self.keySize["SIZE_192"]: + nbrRounds = 12 + elif size == self.keySize["SIZE_256"]: + nbrRounds = 14 + else: + return None + + # the expanded keySize + expandedKeySize = 16 * (nbrRounds + 1) + + # Set the block values, for the block: + # a0,0 a0,1 a0,2 a0,3 + # a1,0 a1,1 a1,2 a1,3 + # a2,0 a2,1 a2,2 a2,3 + # a3,0 a3,1 a3,2 a3,3 + # the mapping order is a0,0 a1,0 a2,0 a3,0 a0,1 a1,1 ... a2,3 a3,3 + + # iterate over the columns + for i in range(4): + # iterate over the rows + for j in range(4): + block[(i + (j * 4))] = iput[(i * 4) + j] + # expand the key into an 176, 208, 240 bytes key + expandedKey = self.expandKey(key, size, expandedKeySize) + # decrypt the block using the expandedKey + block = self.aes_invMain(block, expandedKey, nbrRounds) + # unmap the block again into the output + for k in range(4): + # iterate over the rows + for l in range(4): + output[(k * 4) + l] = block[(k + (l * 4))] + return output + + +class AESModeOfOperation(object): + aes = AES() + + # structure of supported modes of operation + modeOfOperation = dict(OFB=0, CFB=1, CBC=2) + + # converts a 16 character string into a number array + def convertString(self, string, start, end, mode): + if end - start > 16: end = start + 16 + if mode == self.modeOfOperation["CBC"]: + ar = [0] * 16 + else: + ar = [] + + i = start + j = 0 + while len(ar) < end - start: + ar.append(0) + while i < end: + ar[j] = ord(string[i]) + j += 1 + i += 1 + return ar + + # Mode of Operation Encryption + # stringIn - Input String + # mode - mode of type modeOfOperation + # hexKey - a hex key of the bit length size + # size - the bit length of the key + # hexIV - the 128 bit hex Initilization Vector + def encrypt(self, stringIn, mode, key, size, IV): + if len(key) % size: + return None + if len(IV) % 16: + return None + # the AES input/output + plaintext = [] + iput = [0] * 16 + output = [] + ciphertext = [0] * 16 + # the output cipher string + cipherOut = [] + # char firstRound + firstRound = True + if stringIn is not None: + for j in range(int(math.ceil(float(len(stringIn)) / 16))): + start = j * 16 + end = j * 16 + 16 + if end > len(stringIn): + end = len(stringIn) + plaintext = self.convertString(stringIn, start, end, mode) + # print 'PT@%s:%s' % (j, plaintext) + if mode == self.modeOfOperation["CFB"]: + if firstRound: + output = self.aes.encrypt(IV, key, size) + firstRound = False + else: + output = self.aes.encrypt(iput, key, size) + for i in range(16): + if len(plaintext) - 1 < i: + ciphertext[i] = 0 ^ output[i] + elif len(output) - 1 < i: + ciphertext[i] = plaintext[i] ^ 0 + elif len(plaintext) - 1 < i and len(output) < i: + ciphertext[i] = 0 ^ 0 + else: + ciphertext[i] = plaintext[i] ^ output[i] + for k in range(end - start): + cipherOut.append(ciphertext[k]) + iput = ciphertext + elif mode == self.modeOfOperation["OFB"]: + if firstRound: + output = self.aes.encrypt(IV, key, size) + firstRound = False + else: + output = self.aes.encrypt(iput, key, size) + for i in range(16): + if len(plaintext) - 1 < i: + ciphertext[i] = 0 ^ output[i] + elif len(output) - 1 < i: + ciphertext[i] = plaintext[i] ^ 0 + elif len(plaintext) - 1 < i and len(output) < i: + ciphertext[i] = 0 ^ 0 + else: + ciphertext[i] = plaintext[i] ^ output[i] + for k in range(end - start): + cipherOut.append(ciphertext[k]) + iput = output + elif mode == self.modeOfOperation["CBC"]: + for i in range(16): + if firstRound: + iput[i] = plaintext[i] ^ IV[i] + else: + iput[i] = plaintext[i] ^ ciphertext[i] + # print 'IP@%s:%s' % (j, iput) + firstRound = False + ciphertext = self.aes.encrypt(iput, key, size) + # always 16 bytes because of the padding for CBC + for k in range(16): + cipherOut.append(ciphertext[k]) + return mode, len(stringIn), cipherOut + + # Mode of Operation Decryption + # cipherIn - Encrypted String + # originalsize - The unencrypted string length - required for CBC + # mode - mode of type modeOfOperation + # key - a number array of the bit length size + # size - the bit length of the key + # IV - the 128 bit number array Initilization Vector + def decrypt(self, cipherIn, originalsize, mode, key, size, IV): + # cipherIn = unescCtrlChars(cipherIn) + if len(key) % size: + return None + if len(IV) % 16: + return None + # the AES input/output + ciphertext = [] + iput = [] + output = [] + plaintext = [0] * 16 + # the output plain text string + stringOut = '' + # char firstRound + firstRound = True + if cipherIn is not None: + for j in range(int(math.ceil(float(len(cipherIn)) / 16))): + start = j * 16 + end = j * 16 + 16 + if j * 16 + 16 > len(cipherIn): + end = len(cipherIn) + ciphertext = cipherIn[start:end] + if mode == self.modeOfOperation["CFB"]: + if firstRound: + output = self.aes.encrypt(IV, key, size) + firstRound = False + else: + output = self.aes.encrypt(iput, key, size) + for i in range(16): + if len(output) - 1 < i: + plaintext[i] = 0 ^ ciphertext[i] + elif len(ciphertext) - 1 < i: + plaintext[i] = output[i] ^ 0 + elif len(output) - 1 < i and len(ciphertext) < i: + plaintext[i] = 0 ^ 0 + else: + plaintext[i] = output[i] ^ ciphertext[i] + for k in range(end - start): + stringOut += chr(plaintext[k]) + iput = ciphertext + elif mode == self.modeOfOperation["OFB"]: + if firstRound: + output = self.aes.encrypt(IV, key, size) + firstRound = False + else: + output = self.aes.encrypt(iput, key, size) + for i in range(16): + if len(output) - 1 < i: + plaintext[i] = 0 ^ ciphertext[i] + elif len(ciphertext) - 1 < i: + plaintext[i] = output[i] ^ 0 + elif len(output) - 1 < i and len(ciphertext) < i: + plaintext[i] = 0 ^ 0 + else: + plaintext[i] = output[i] ^ ciphertext[i] + for k in range(end - start): + stringOut += chr(plaintext[k]) + iput = output + elif mode == self.modeOfOperation["CBC"]: + output = self.aes.decrypt(ciphertext, key, size) + for i in range(16): + if firstRound: + plaintext[i] = IV[i] ^ output[i] + else: + plaintext[i] = iput[i] ^ output[i] + firstRound = False + if originalsize is not None and originalsize < end: + for k in range(originalsize - start): + stringOut += chr(plaintext[k]) + else: + for k in range(end - start): + stringOut += chr(plaintext[k]) + iput = ciphertext + return stringOut + + +def encryptData(key, data, mode=AESModeOfOperation.modeOfOperation["CBC"]): + """encrypt `data` using `key` + + `key` should be a string of bytes. + + returned cipher is a string of bytes prepended with the initialization + vector. + + """ + key = map(ord, key) + if mode == AESModeOfOperation.modeOfOperation["CBC"]: + data = append_PKCS7_padding(data) + keysize = len(key) + assert keysize in AES.keySize.values(), 'invalid key size: %s' % keysize + # create a new iv using random data + iv = [ord(i) for i in os.urandom(16)] + moo = AESModeOfOperation() + (mode, length, ciph) = moo.encrypt(data, mode, key, keysize, iv) + # With padding, the original length does not need to be known. It's a bad + # idea to store the original message length. + # prepend the iv. + return ''.join(map(chr, iv)) + ''.join(map(chr, ciph)) + + +def decryptData(key, data, mode=AESModeOfOperation.modeOfOperation["CBC"]): + """decrypt `data` using `key` + + `key` should be a string of bytes. + + `data` should have the initialization vector prepended as a string of + ordinal values. + + """ + + key = map(ord, key) + keysize = len(key) + assert keysize in AES.keySize.values(), 'invalid key size: %s' % keysize + # iv is first 16 bytes + iv = map(ord, data[:16]) + data = map(ord, data[16:]) + moo = AESModeOfOperation() + decr = moo.decrypt(data, None, mode, key, keysize, iv) + if mode == AESModeOfOperation.modeOfOperation["CBC"]: + decr = strip_PKCS7_padding(decr) + return decr + +if __name__ == "__main__": + moo = AESModeOfOperation() + cleartext = "This is a test!" + cypherkey = [143, 194, 34, 208, 145, 203, 230, 143, 177, 246, 97, 206, 145, + 92, 255, 84] + iv = [103, 35, 148, 239, 76, 213, 47, 118, 255, 222, 123, 176, 106, 134, 98, + 92] + mode, orig_len, ciph = moo.encrypt(cleartext, moo.modeOfOperation["CBC"], + cypherkey, moo.aes.keySize["SIZE_128"], + iv) + print 'm=%s, ol=%s (%s), ciph=%s' % (mode, orig_len, len(cleartext), ciph) + decr = moo.decrypt(ciph, orig_len, mode, cypherkey, + moo.aes.keySize["SIZE_128"], iv) + print decr diff --git a/client/socks.py b/client/socks.py new file mode 100644 index 0000000..af32842 --- /dev/null +++ b/client/socks.py @@ -0,0 +1,410 @@ +"""SocksiPy - Python SOCKS module. +Version 1.00 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +""" + +import socket +import struct +import random + +PROXY_TYPE_SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = 2 +PROXY_TYPE_HTTP = 3 + +_defaultproxy = None +_orgsocket = socket.socket + + +class ProxyError(IOError): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +class GeneralProxyError(ProxyError): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +class Socks5AuthError(ProxyError): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +class Socks5Error(ProxyError): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +class Socks4Error(ProxyError): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +class HTTPError(ProxyError): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +_generalerrors = ("success", "invalid data", "not connected", "not available", + "bad proxy type", "bad input") + +_socks5errors = ("succeeded", "general SOCKS server failure", + "connection not allowed by ruleset", "Network unreachable", + "Host unreachable", "Connection refused", "TTL expired", + "Command not supported", "Address type not supported", + "Unknown error") + +_socks5autherrors = ("succeeded", "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", "unknown error") + +_socks4errors = ( + "request granted", "request rejected or failed", + "request rejected because SOCKS server cannot connect to identd on the client", + "request rejected because the client program and identd report different user-ids", + "unknown error") + + +def setdefaultproxy(proxytype=None, + addr=None, + port=None, + rdns=True, + username=str(random.randrange(10000000, 99999999)), + password=str(random.randrange(10000000, 99999999))): + """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. + """ + global _defaultproxy + _defaultproxy = (proxytype, addr, port, rdns, username, password) + + +class socksocket(socket.socket): + """socksocket([family[, type[, proto]]]) -> socket object + + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET, type=SOCK_STREAM and proto=0. + """ + + def __init__(self, + family=socket.AF_INET, + type=socket.SOCK_STREAM, + proto=0, + _sock=None): + _orgsocket.__init__(self, family, type, proto, _sock) + if _defaultproxy is not None: + self.__proxy = _defaultproxy + else: + self.__proxy = (None, None, None, None, None, None) + self.__proxysockname = None + self.__proxypeername = None + + def __recvall(self, bytes): + """__recvall(bytes) -> data + Receive EXACTLY the number of bytes requested from the socket. + Blocks until the required number of bytes have been received. + """ + data = "" + while len(data) < bytes: + data = data + self.recv(bytes - len(data)) + return data + + def setproxy(self, + proxytype=None, + addr=None, + port=None, + rdns=True, + username=None, + password=None): + """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + proxytype - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be preformed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.__proxy = (proxytype, addr, port, rdns, username, password) + + def __negotiatesocks5(self, destaddr, destport): + """__negotiatesocks5(self,destaddr,destport) + Negotiates a connection through a SOCKS5 server. + """ + # First we'll send the authentication packages we support. + if (self.__proxy[4] is not None) and (self.__proxy[5] is not None): + # The username/password details were supplied to the + # setproxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + self.sendall("\x05\x02\x00\x02") + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + self.sendall("\x05\x01\x00") + # We'll receive the server's response to determine which + # method was selected + chosenauth = self.__recvall(2) + if chosenauth[0] != "\x05": + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + # Check the chosen authentication method + if chosenauth[1] == "\x00": + # No authentication is required + pass + elif chosenauth[1] == "\x02": + # Okay, we need to perform a basic username/password + # authentication. + self.sendall("\x01" + chr(len(self.__proxy[4])) + self.__proxy[4] + + chr(len(self.__proxy[5])) + self.__proxy[5]) + authstat = self.__recvall(2) + if authstat[0] != "\x01": + # Bad response + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if authstat[1] != "\x00": + # Authentication failed + self.close() + raise Socks5AuthError, (3, _socks5autherrors[3]) + # Authentication succeeded + else: + # Reaching here is always bad + self.close() + if chosenauth[1] == "\xFF": + raise Socks5AuthError((2, _socks5autherrors[2])) + else: + raise GeneralProxyError((1, _generalerrors[1])) + # Now we can request the actual connection + req = "\x05\x01\x00" + # If the given destination address is an IP address, we'll + # use the IPv4 address request even if remote resolving was specified. + try: + ipaddr = socket.inet_aton(destaddr) + req = req + "\x01" + ipaddr + except socket.error: + # Well it's not an IP number, so it's probably a DNS name. + if self.__proxy[3]: + # Resolve remotely + ipaddr = None + req = req + "\x03" + chr(len(destaddr)) + destaddr + else: + # Resolve locally + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + req = req + "\x01" + ipaddr + req += struct.pack(">H", destport) + self.sendall(req) + # Get the response + resp = self.__recvall(4) + if resp[0] != "\x05": + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + elif resp[1] != "\x00": + # Connection failed + self.close() + raise Socks5Error(_socks5errors[min(9, ord(resp[1]))]) + # Get the bound address/port + elif resp[3] == "\x01": + boundaddr = self.__recvall(4) + elif resp[3] == "\x03": + resp = resp + self.recv(1) + boundaddr = self.__recvall(resp[4]) + else: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + boundport = struct.unpack(">H", self.__recvall(2))[0] + self.__proxysockname = (boundaddr, boundport) + if ipaddr is not None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def getproxysockname(self): + """getsockname() -> address info + Returns the bound IP address and port number at the proxy. + """ + return self.__proxysockname + + def getproxypeername(self): + """getproxypeername() -> address info + Returns the IP and port number of the proxy. + """ + return _orgsocket.getpeername(self) + + def getpeername(self): + """getpeername() -> address info + Returns the IP address and port number of the destination + machine (note: getproxypeername returns the proxy) + """ + return self.__proxypeername + + def __negotiatesocks4(self, destaddr, destport): + """__negotiatesocks4(self,destaddr,destport) + Negotiates a connection through a SOCKS4 server. + """ + # Check if the destination address provided is an IP address + rmtrslv = False + try: + ipaddr = socket.inet_aton(destaddr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if self.__proxy[3]: + ipaddr = "\x00\x00\x00\x01" + rmtrslv = True + else: + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + # Construct the request packet + req = "\x04\x01" + struct.pack(">H", destport) + ipaddr + # The username parameter is considered userid for SOCKS4 + if self.__proxy[4] is not None: + req = req + self.__proxy[4] + req += "\x00" + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if rmtrslv: + req = req + destaddr + "\x00" + self.sendall(req) + # Get the response from the server + resp = self.__recvall(8) + if resp[0] != "\x00": + # Bad data + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if resp[1] != "\x5A": + # Server returned an error + self.close() + if ord(resp[1]) in (91, 92, 93): + self.close() + raise Socks4Error((ord(resp[1]), _socks4errors[ord(resp[1]) - + 90])) + else: + raise Socks4Error((94, _socks4errors[4])) + # Get the bound address/port + self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack( + ">H", resp[2:4])[0]) + if rmtrslv is not None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def __negotiatehttp(self, destaddr, destport): + """__negotiatehttp(self,destaddr,destport) + Negotiates a connection through an HTTP server. + """ + # If we need to resolve locally, we do this now + if not self.__proxy[3]: + addr = socket.gethostbyname(destaddr) + else: + addr = destaddr + self.sendall("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + + "Host: " + destaddr + "\r\n\r\n") + # We read the response until we get the string "\r\n\r\n" + resp = self.recv(1) + while resp.find("\r\n\r\n") == -1: + resp = resp + self.recv(1) + # We just need the first line to check if the connection + # was successful + statusline = resp.splitlines()[0].split(" ", 2) + if statusline[0] not in ("HTTP/1.0", "HTTP/1.1"): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + try: + statuscode = int(statusline[1]) + except ValueError: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if statuscode != 200: + self.close() + raise HTTPError((statuscode, statusline[2])) + self.__proxysockname = ("0.0.0.0", 0) + self.__proxypeername = (addr, destport) + + def connect(self, destpair): + """connect(self,despair) + Connects to the specified destination through a proxy. + destpar - A tuple of the IP/DNS address and the port number. + (identical to socket's connect). + To select the proxy server use setproxy(). + """ + # Do a minimal input check first + if (type(destpair) in + (list, tuple) == False) or (len(destpair) < 2) or ( + type(destpair[0]) != str) or (type(destpair[1]) != int): + raise GeneralProxyError((5, _generalerrors[5])) + if self.__proxy[0] == PROXY_TYPE_SOCKS5: + if self.__proxy[2] is not None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatesocks5(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_SOCKS4: + if self.__proxy[2] is not None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatesocks4(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP: + if self.__proxy[2] is not None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatehttp(destpair[0], destpair[1]) + elif self.__proxy[0] is None: + _orgsocket.connect(self, (destpair[0], destpair[1])) + else: + raise GeneralProxyError((4, _generalerrors[4])) diff --git a/client/support.py b/client/support.py new file mode 100644 index 0000000..242bac0 --- /dev/null +++ b/client/support.py @@ -0,0 +1,429 @@ +from __future__ import absolute_import, print_function + +import sys + +import logging +import pprint +import random + +from decimal import Decimal + +from math import exp + +# todo: this was the date format used in the original debug(). Use it? +# logging.basicConfig(filename='logs/joinmarket.log', +# stream=sys.stdout, +# level=logging.DEBUG, +# format='%(asctime)s %(message)s', +# dateformat='[%Y/%m/%d %H:%M:%S] ') + +logFormatter = logging.Formatter( + "%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s") +log = logging.getLogger('joinmarket') +log.setLevel(logging.DEBUG) + +ORDER_KEYS = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', + 'cjfee'] + +joinmarket_alert = [''] +core_alert = [''] +debug_silence = [False] + + +#consoleHandler = logging.StreamHandler(stream=sys.stdout) +class JoinMarketStreamHandler(logging.StreamHandler): + + def __init__(self, stream): + super(JoinMarketStreamHandler, self).__init__(stream) + + def emit(self, record): + if joinmarket_alert[0]: + print('JoinMarket Alert Message: ' + joinmarket_alert[0]) + if core_alert[0]: + print('Core Alert Message: ' + core_alert[0]) + if not debug_silence[0]: + super(JoinMarketStreamHandler, self).emit(record) + + +consoleHandler = JoinMarketStreamHandler(stream=sys.stdout) +consoleHandler.setFormatter(logFormatter) +log.addHandler(consoleHandler) + +# log = logging.getLogger('joinmarket') +# log.addHandler(logging.NullHandler()) + +log.debug('hello joinmarket') + + +def get_log(): + """ + provides joinmarket logging instance + :return: log instance + """ + return log + + +""" +Random functions - replacing some NumPy features +NOTE THESE ARE NEITHER CRYPTOGRAPHICALLY SECURE +NOR PERFORMANT NOR HIGH PRECISION! +Only for sampling purposes +""" + + +def rand_norm_array(mu, sigma, n): + # use normalvariate instead of gauss for thread safety + return [random.normalvariate(mu, sigma) for _ in range(n)] + + +def rand_exp_array(lamda, n): + # 'lambda' is reserved (in case you are triggered by spelling errors) + return [random.expovariate(1.0 / lamda) for _ in range(n)] + + +def rand_pow_array(power, n): + # rather crude in that uses a uniform sample which is a multiple of 1e-4 + # for basis of formula, see: http://mathworld.wolfram.com/RandomNumber.html + return [y**(1.0 / power) + for y in [x * 0.0001 for x in random.sample( + xrange(10000), n)]] + + +def rand_weighted_choice(n, p_arr): + """ + Choose a value in 0..n-1 + with the choice weighted by the probabilities + in the list p_arr. Note that there will be some + floating point rounding errors, but see the note + at the top of this section. + """ + if abs(sum(p_arr) - 1.0) > 1e-4: + raise ValueError("Sum of probabilities must be 1") + if len(p_arr) != n: + raise ValueError("Need: " + str(n) + " probabilities.") + cum_pr = [sum(p_arr[:i + 1]) for i in xrange(len(p_arr))] + r = random.random() + return sorted(cum_pr + [r]).index(r) + +# End random functions + + +def chunks(d, n): + return [d[x:x + n] for x in xrange(0, len(d), n)] + +def select(unspent, value): + """Default coin selection algorithm. + """ + value = int(value) + high = [u for u in unspent if u["value"] >= value] + high.sort(key=lambda u: u["value"]) + low = [u for u in unspent if u["value"] < value] + low.sort(key=lambda u: -u["value"]) + if len(high): + return [high[0]] + i, tv = 0, 0 + while tv < value and i < len(low): + tv += low[i]["value"] + i += 1 + if tv < value: + raise Exception("Not enough funds") + return low[:i] + +def select_gradual(unspent, value): + """ + UTXO selection algorithm for gradual dust reduction + If possible, combines outputs, picking as few as possible of the largest + utxos less than the target value; if the target value is larger than the + sum of all smaller utxos, uses the smallest utxo larger than the value. + """ + value, key = int(value), lambda u: u["value"] + high = sorted([u for u in unspent if key(u) >= value], key=key) + low = sorted([u for u in unspent if key(u) < value], key=key) + lowsum = reduce(lambda x, y: x + y, map(key, low), 0) + if value > lowsum: + if len(high) == 0: + raise Exception('Not enough funds') + else: + return [high[0]] + else: + start, end, total = 0, 0, 0 + while total < value: + total += low[end]['value'] + end += 1 + while total >= value + low[start]['value']: + total -= low[start]['value'] + start += 1 + return low[start:end] + + +def select_greedy(unspent, value): + """ + UTXO selection algorithm for greedy dust reduction, but leaves out + extraneous utxos, preferring to keep multiple small ones. + """ + value, key, cursor = int(value), lambda u: u['value'], 0 + utxos, picked = sorted(unspent, key=key), [] + for utxo in utxos: # find the smallest consecutive sum >= value + value -= key(utxo) + if value == 0: # perfect match! (skip dilution stage) + return utxos[0:cursor + 1] # end is non-inclusive + elif value < 0: # overshot + picked += [utxo] # definitely need this utxo + break # proceed to dilution + cursor += 1 + for utxo in utxos[cursor - 1::-1]: # dilution loop + value += key(utxo) # see if we can skip this one + if value > 0: # no, that drops us below the target + picked += [utxo] # so we need this one too + value -= key(utxo) # 'backtrack' the counter + if len(picked) > 0: + return picked + raise Exception('Not enough funds') # if all else fails, we do too + + +def select_greediest(unspent, value): + """ + UTXO selection algorithm for speediest dust reduction + Combines the shortest run of utxos (sorted by size, from smallest) which + exceeds the target value; if the target value is larger than the sum of + all smaller utxos, uses the smallest utxo larger than the target value. + """ + value, key = int(value), lambda u: u["value"] + high = sorted([u for u in unspent if key(u) >= value], key=key) + low = sorted([u for u in unspent if key(u) < value], key=key) + lowsum = reduce(lambda x, y: x + y, map(key, low), 0) + if value > lowsum: + if len(high) == 0: + raise Exception('Not enough funds') + else: + return [high[0]] + else: + end, total = 0, 0 + while total < value: + total += low[end]['value'] + end += 1 + return low[0:end] + + +def calc_cj_fee(ordertype, cjfee, cj_amount): + if ordertype == 'absoffer': + real_cjfee = int(cjfee) + elif ordertype == 'reloffer': + real_cjfee = int((Decimal(cjfee) * Decimal(cj_amount)).quantize(Decimal( + 1))) + else: + raise RuntimeError('unknown order type: ' + str(ordertype)) + return real_cjfee + + +def weighted_order_choose(orders, n): + """ + Algorithm for choosing the weighting function + it is an exponential + P(f) = exp(-(f - fmin) / phi) + P(f) - probability of order being chosen + f - order fee + fmin - minimum fee in the order book + phi - scaling parameter, 63% of the distribution is within + + define number M, related to the number of counterparties in this coinjoin + phi has a value such that it contains up to the Mth order + unless M < orderbook size, then phi goes up to the last order + """ + minfee = orders[0][1] + M = int(3 * n) + if len(orders) > M: + phi = orders[M][1] - minfee + else: + phi = orders[-1][1] - minfee + fee = [o[1] for o in orders] + if phi > 0: + weight = [exp(-(1.0 * f - minfee) / phi) for f in fee] + else: + weight = [1.0] * len(fee) + weight = [x / sum(weight) for x in weight] + log.debug('phi=' + str(phi) + ' weights = ' + str(weight)) + chosen_order_index = rand_weighted_choice(len(orders), weight) + return orders[chosen_order_index] + + +def cheapest_order_choose(orders, n): + """ + Return the cheapest order from the orders. + """ + return orders[0] + + +def pick_order(orders, n): + print("Considered orders:") + for i, o in enumerate(orders): + print(" %2d. %20s, CJ fee: %6s, tx fee: %6d" % + (i, o[0]['counterparty'], str(o[0]['cjfee']), o[0]['txfee'])) + pickedOrderIndex = -1 + if i == 0: + print("Only one possible pick, picking it.") + return orders[0] + while pickedOrderIndex == -1: + try: + pickedOrderIndex = int(raw_input('Pick an order between 0 and ' + + str(i) + ': ')) + except ValueError: + pickedOrderIndex = -1 + continue + + if 0 <= pickedOrderIndex < len(orders): + return orders[pickedOrderIndex] + pickedOrderIndex = -1 + + +def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None): + if ignored_makers is None: + ignored_makers = [] + #Filter ignored makers and inappropriate amounts + orders = [o for o in offers if o['counterparty'] not in ignored_makers] + orders = [o for o in orders if o['minsize'] < cj_amount] + orders = [o for o in orders if o['maxsize'] > cj_amount] + orders_fees = [( + o, calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount) - o['txfee']) + for o in orders] + + counterparties = set([o['counterparty'] for o in orders]) + if n > len(counterparties): + log.debug(('ERROR not enough liquidity in the orderbook n=%d ' + 'suitable-counterparties=%d amount=%d totalorders=%d') % + (n, len(counterparties), cj_amount, len(orders))) + # TODO handle not enough liquidity better, maybe an Exception + return None, 0 + """ + restrict to one order per counterparty, choose the one with the lowest + cjfee this is done in advance of the order selection algo, so applies to + all of them. however, if orders are picked manually, allow duplicates. + """ + feekey = lambda x: x[1] + if chooseOrdersBy != pick_order: + orders_fees = sorted( + dict((v[0]['counterparty'], v) + for v in sorted(orders_fees, + key=feekey, + reverse=True)).values(), + key=feekey) + else: + orders_fees = sorted(orders_fees, key=feekey) #sort by ascending cjfee + + log.debug('considered orders = \n' + '\n'.join([str(o) for o in orders_fees + ])) + total_cj_fee = 0 + chosen_orders = [] + for i in range(n): + chosen_order, chosen_fee = chooseOrdersBy(orders_fees, n) + # remove all orders from that same counterparty + orders_fees = [o + for o in orders_fees + if o[0]['counterparty'] != chosen_order['counterparty']] + chosen_orders.append(chosen_order) + total_cj_fee += chosen_fee + log.debug('chosen orders = \n' + '\n'.join([str(o) for o in chosen_orders])) + result = dict([(o['counterparty'], o) for o in chosen_orders]) + return result, total_cj_fee + + +def choose_sweep_orders(db, + total_input_value, + txfee, + n, + chooseOrdersBy, + ignored_makers=None): + """ + choose an order given that we want to be left with no change + i.e. sweep an entire group of utxos + + solve for cjamount when mychange = 0 + for an order with many makers, a mixture of absoffer and reloffer + mychange = totalin - cjamount - total_txfee - sum(absfee) - sum(relfee*cjamount) + => 0 = totalin - mytxfee - sum(absfee) - cjamount*(1 + sum(relfee)) + => cjamount = (totalin - mytxfee - sum(absfee)) / (1 + sum(relfee)) + """ + total_txfee = txfee * n + + if ignored_makers is None: + ignored_makers = [] + + def calc_zero_change_cj_amount(ordercombo): + sumabsfee = 0 + sumrelfee = Decimal('0') + sumtxfee_contribution = 0 + for order in ordercombo: + sumtxfee_contribution += order['txfee'] + if order['ordertype'] == 'absoffer': + sumabsfee += int(order['cjfee']) + elif order['ordertype'] == 'reloffer': + sumrelfee += Decimal(order['cjfee']) + else: + raise RuntimeError('unknown order type: {}'.format(order[ + 'ordertype'])) + + my_txfee = max(total_txfee - sumtxfee_contribution, 0) + cjamount = (total_input_value - my_txfee - sumabsfee) / (1 + sumrelfee) + cjamount = int(cjamount.quantize(Decimal(1))) + return cjamount, int(sumabsfee + sumrelfee * cjamount) + + log.debug('choosing sweep orders for total_input_value = ' + str( + total_input_value) + ' n=' + str(n)) + sqlorders = db.execute('SELECT * FROM orderbook WHERE minsize <= ?;', + (total_input_value,)).fetchall() + orderlist = [dict([(k, o[k]) for k in ORDER_KEYS]) + for o in sqlorders if o['counterparty'] not in ignored_makers] + + log.debug('orderlist = \n' + '\n'.join([str(o) for o in orderlist])) + orders_fees = [(o, calc_cj_fee(o['ordertype'], o['cjfee'], + total_input_value)) for o in orderlist] + + feekey = lambda x: x[1] + # sort from smallest to biggest cj fee + orders_fees = sorted(orders_fees, key=feekey) + chosen_orders = [] + while len(chosen_orders) < n: + for i in range(n - len(chosen_orders)): + if len(orders_fees) < n - len(chosen_orders): + log.debug('ERROR not enough liquidity in the orderbook') + # TODO handle not enough liquidity better, maybe an Exception + return None, 0, 0 + chosen_order, chosen_fee = chooseOrdersBy(orders_fees, n) + log.debug('chosen = ' + str(chosen_order)) + # remove all orders from that same counterparty + orders_fees = [ + o + for o in orders_fees + if o[0]['counterparty'] != chosen_order['counterparty'] + ] + chosen_orders.append(chosen_order) + # calc cj_amount and check its in range + cj_amount, total_fee = calc_zero_change_cj_amount(chosen_orders) + for c in list(chosen_orders): + minsize = c['minsize'] + maxsize = c['maxsize'] + if cj_amount > maxsize or cj_amount < minsize: + chosen_orders.remove(c) + log.debug('chosen orders = \n' + '\n'.join([str(o) for o in chosen_orders])) + result = dict([(o['counterparty'], o) for o in chosen_orders]) + log.debug('cj amount = ' + str(cj_amount)) + return result, cj_amount, total_fee + + +def debug_dump_object(obj, skip_fields=None): + if skip_fields is None: + skip_fields = [] + log.debug('Class debug dump, name:' + obj.__class__.__name__) + for k, v in obj.__dict__.iteritems(): + if k in skip_fields: + continue + if k == 'password' or k == 'given_password': + continue + log.debug('key=' + k) + if isinstance(v, str): + log.debug('string: len:' + str(len(v))) + log.debug(v) + elif isinstance(v, dict) or isinstance(v, list): + log.debug(pprint.pformat(v)) + else: + log.debug(str(v)) diff --git a/client/taker.py b/client/taker.py new file mode 100644 index 0000000..0fb309f --- /dev/null +++ b/client/taker.py @@ -0,0 +1,562 @@ +#! /usr/bin/env python +from __future__ import print_function + +import base64 +import pprint +import random +import sys +import time +import copy + +import btc +from joinmarketclient.configure import jm_single, get_p2pk_vbyte, donation_address +from joinmarketclient.support import (get_log, calc_cj_fee, weighted_order_choose, + choose_orders) +from joinmarketclient.wallet import estimate_tx_fee +from joinmarketclient.podle import (generate_podle, get_podle_commitments, + PoDLE, PoDLEError) +jlog = get_log() + + +class JMTakerError(Exception): + pass + +#Taker is now a class to do 1 coinjoin +class Taker(object): + + def __init__(self, + wallet, + mixdepth, + amount, + n_counterparties, + order_chooser=weighted_order_choose, + external_addr=None, + sign_method=None, + callbacks=None): + self.wallet = wallet + self.mixdepth = mixdepth + self.cjamount = amount + self.my_cj_addr = external_addr + self.order_chooser = order_chooser + self.n_counterparties = n_counterparties + self.ignored_makers = None + self.outputs = [] + self.cjfee_total = 0 + self.maker_txfee_contributions = 0 + self.txfee_default = 5000 + self.txid = None + #allow custom wallet-based clients to use their own signing code; + #currently only setting "wallet" is allowed, calls wallet.sign_tx(tx) + self.sign_method = sign_method + if callbacks: + self.filter_orders_callback, self.taker_info_callback = callbacks + else: + self.filter_orders_callback = None + self.taker_info_callback = self.default_taker_info_callback + + def default_taker_info_callback(self, infotype, msg): + jlog.debug(infotype + ":" + msg) + + def initialize(self, orderbook): + """Once the daemon is active and has returned the current orderbook, + select offers and prepare a commitment, then send it to the protocol + to fill offers. + """ + if not self.filter_orderbook(orderbook): + return (False,) + #choose coins to spend + if not self.prepare_my_bitcoin_data(): + return (False,) + #Prepare a commitment + commitment, revelation, errmsg = self.make_commitment() + if not commitment: + self.taker_info_callback("ABORT", errmsg) + return (False,) + else: + self.taker_info_callback("INFO", errmsg) + return (True, self.cjamount, commitment, revelation, self.orderbook) + + def filter_orderbook(self, orderbook): + self.orderbook, self.total_cj_fee = choose_orders( + orderbook, self.cjamount, self.n_counterparties, self.order_chooser, + self.ignored_makers) + if self.filter_orders_callback: + accepted = self.filter_orders_callback([self.orderbook, + self.total_cj_fee]) + if not accepted: + return False + return True + + def prepare_my_bitcoin_data(self): + """Get a coinjoin address and a change address; prepare inputs + appropriate for this transaction""" + if not self.my_cj_addr: + try: + self.my_cj_addr = self.wallet.get_external_addr(self.mixdepth + 1) + except: + self.taker_error_callback("ABORT", "Failed to get an address") + return False + self.my_change_addr = None + if self.cjamount != 0: + try: + self.my_change_addr = self.wallet.get_internal_addr(self.mixdepth) + except: + self.taker_error_callback("ABORT", "Failed to get a change address") + return False + #TODO sweep, doesn't apply here + self.total_txfee = 2 * self.txfee_default * self.n_counterparties + total_amount = self.cjamount + self.total_cj_fee + self.total_txfee + jlog.debug('total estimated amount spent = ' + str(total_amount)) + #adjust the required amount upwards to anticipate an increase in + #transaction fees after re-estimation; this is sufficiently conservative + #to make failures unlikely while keeping the occurence of failure to + #find sufficient utxos extremely rare. Indeed, a doubling of 'normal' + #txfee indicates undesirable behaviour on maker side anyway. + try: + self.input_utxos = self.wallet.select_utxos(self.mixdepth, + total_amount) + except Exception as e: + self.taker_error_callback("ABORT", + "Unable to select sufficient coins: " + repr(e)) + return False + self.utxos = {None: self.input_utxos.keys()} + return True + + def receive_utxos(self, ioauth_data): + """Triggered when the daemon returns utxo data from + makers who responded; this is the completion of phase 1 + of the protocol + """ + rejected_counterparties = [] + #Enough data, but need to authorize against the btc pubkey first. + for nick, nickdata in ioauth_data.iteritems(): + utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata + if not self.auth_counterparty(btc_sig, auth_pub, maker_pk): + print("Counterparty encryption verification failed, aborting") + #This counterparty must be rejected + rejected_counterparties.append(nick) + + for rc in rejected_counterparties: + del ioauth_data[rc] + + self.maker_utxo_data = {} + + for nick, nickdata in ioauth_data.iteritems(): + utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata + self.utxos[nick] = utxo_list + utxo_data = jm_single().bc_interface.query_utxo_set(self.utxos[ + nick]) + if None in utxo_data: + jlog.debug(('ERROR outputs unconfirmed or already spent. ' + 'utxo_data={}').format(pprint.pformat(utxo_data))) + # when internal reviewing of makers is created, add it here to + # immediately quit; currently, the timeout thread suffices. + continue + + #Complete maker authorization: + #Extract the address fields from the utxos + #Construct the Bitcoin address for the auth_pub field + #Ensure that at least one address from utxos corresponds. + input_addresses = [d['address'] for d in utxo_data] + auth_address = btc.pubkey_to_address(auth_pub, get_p2pk_vbyte()) + if not auth_address in input_addresses: + jlog.warn("ERROR maker's (" + nick + ")" + " authorising pubkey is not included " + "in the transaction: " + str(auth_address)) + #this will not be added to the transaction, so we will have + #to recheck if we have enough + continue + + total_input = sum([d['value'] for d in utxo_data]) + real_cjfee = calc_cj_fee(self.orderbook[nick]['ordertype'], + self.orderbook[nick]['cjfee'], + self.cjamount) + change_amount = (total_input - self.cjamount - + self.orderbook[nick]['txfee'] + real_cjfee) + + # certain malicious and/or incompetent liquidity providers send + # inputs totalling less than the coinjoin amount! this leads to + # a change output of zero satoshis; this counterparty must be removed. + if change_amount < jm_single().DUST_THRESHOLD: + fmt = ('ERROR counterparty requires sub-dust change. nick={}' + 'totalin={:d} cjamount={:d} change={:d}').format + jlog.debug(fmt(nick, total_input, self.cjamount, change_amount)) + jlog.warn("Invalid change, too small, nick= " + nick) + continue + + self.outputs.append({'address': change_addr, + 'value': change_amount}) + fmt = ('fee breakdown for {} totalin={:d} ' + 'cjamount={:d} txfee={:d} realcjfee={:d}').format + jlog.debug(fmt(nick, total_input, self.cjamount, self.orderbook[ + nick]['txfee'], real_cjfee)) + self.outputs.append({'address': cj_addr, 'value': self.cjamount}) + self.cjfee_total += real_cjfee + self.maker_txfee_contributions += self.orderbook[nick]['txfee'] + self.maker_utxo_data[nick] = utxo_data + + #Apply business logic of how many counterparties are enough: + if len(self.maker_utxo_data.keys()) < jm_single().config.getint( + "POLICY", "minimum_makers"): + return (False, + "Not enough counterparties responded to fill, giving up") + + jlog.info('got all parts, enough to build a tx') + self.nonrespondants = list(self.maker_utxo_data.keys()) + + my_total_in = sum([va['value'] for u, va in self.input_utxos.iteritems() + ]) + if self.my_change_addr: + #Estimate fee per choice of next/3/6 blocks targetting. + estimated_fee = estimate_tx_fee( + len(sum(self.utxos.values(), [])), len(self.outputs) + 2) + jlog.info("Based on initial guess: " + str(self.total_txfee) + + ", we estimated a miner fee of: " + str(estimated_fee)) + #reset total + self.total_txfee = estimated_fee + my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0) + my_change_value = ( + my_total_in - self.cjamount - self.cjfee_total - my_txfee) + #Since we could not predict the maker's inputs, we may end up needing + #too much such that the change value is negative or small. Note that + #we have tried to avoid this based on over-estimating the needed amount + #in SendPayment.create_tx(), but it is still a possibility if one maker + #uses a *lot* of inputs. + if self.my_change_addr and my_change_value <= 0: + raise ValueError("Calculated transaction fee of: " + str( + self.total_txfee) + + " is too large for our inputs;Please try again.") + elif self.my_change_addr and my_change_value <= jm_single( + ).BITCOIN_DUST_THRESHOLD: + jlog.info("Dynamically calculated change lower than dust: " + str( + my_change_value) + "; dropping.") + self.my_change_addr = None + my_change_value = 0 + jlog.info( + 'fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d' + % (my_total_in, my_txfee, self.maker_txfee_contributions, + self.cjfee_total, my_change_value)) + if self.my_change_addr is None: + if my_change_value != 0 and abs(my_change_value) != 1: + # seems you wont always get exactly zero because of integer + # rounding so 1 satoshi extra or fewer being spent as miner + # fees is acceptable + jlog.debug(('WARNING CHANGE NOT BEING ' + 'USED\nCHANGEVALUE = {}').format(my_change_value)) + else: + self.outputs.append({'address': self.my_change_addr, + 'value': my_change_value}) + self.utxo_tx = [dict([('output', u)]) + for u in sum(self.utxos.values(), [])] + self.outputs.append({'address': self.coinjoin_address(), + 'value': self.cjamount}) + random.shuffle(self.utxo_tx) + random.shuffle(self.outputs) + tx = btc.mktx(self.utxo_tx, self.outputs) + jlog.debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) + + self.latest_tx = btc.deserialize(tx) + for index, ins in enumerate(self.latest_tx['ins']): + utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + if utxo not in self.input_utxos.keys(): + continue + # placeholders required + ins['script'] = 'deadbeef' + + return (True, self.maker_utxo_data.keys(), tx) + + def auth_counterparty(self, btc_sig, auth_pub, maker_pk): + """Validate the counterpartys claim to own the btc + address/pubkey that will be used for coinjoining + with an ecdsa verification. + """ + if not btc.ecdsa_verify(maker_pk, btc_sig, auth_pub): + jlog.debug('signature didnt match pubkey and message') + return False + return True + + def on_sig(self, nick, sigb64): + sig = base64.b64decode(sigb64).encode('hex') + inserted_sig = False + txhex = btc.serialize(self.latest_tx) + + # batch retrieval of utxo data + utxo = {} + ctr = 0 + for index, ins in enumerate(self.latest_tx['ins']): + utxo_for_checking = ins['outpoint']['hash'] + ':' + str(ins[ + 'outpoint']['index']) + if (ins['script'] != '' or + utxo_for_checking in self.input_utxos.keys()): + continue + utxo[ctr] = [index, utxo_for_checking] + ctr += 1 + utxo_data = jm_single().bc_interface.query_utxo_set([x[ + 1] for x in utxo.values()]) + + # insert signatures + for i, u in utxo.iteritems(): + if utxo_data[i] is None: + continue + sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'], + *btc.deserialize_script(sig)) + if sig_good: + jlog.debug('found good sig at index=%d' % (u[0])) + self.latest_tx['ins'][u[0]]['script'] = sig + inserted_sig = True + # check if maker has sent everything possible + self.utxos[nick].remove(u[1]) + if len(self.utxos[nick]) == 0: + jlog.debug(('nick = {} sent all sigs, removing from ' + 'nonrespondant list').format(nick)) + self.nonrespondants.remove(nick) + break + if not inserted_sig: + jlog.debug('signature did not match anything in the tx') + # TODO what if the signature doesnt match anything + # nothing really to do except drop it, carry on and wonder why the + # other guy sent a failed signature + + tx_signed = True + for ins in self.latest_tx['ins']: + if ins['script'] == '': + tx_signed = False + if not tx_signed: + return False + assert not len(self.nonrespondants) + jlog.debug('all makers have sent their signatures') + self.self_sign_and_push() + return True + + def make_commitment(self): + """The Taker default commitment function, which uses PoDLE. + Alternative commitment types should use a different commit type byte. + This will allow future upgrades to provide different style commitments + by subclassing Taker and changing the commit_type_byte; existing makers + will simply not accept this new type of commitment. + In case of success, return the commitment and its opening. + In case of failure returns (None, None) and constructs a detailed + log for the user to read and discern the reason. + """ + + def filter_by_coin_age_amt(utxos, age, amt): + results = jm_single().bc_interface.query_utxo_set(utxos, + includeconf=True) + newresults = [] + too_old = [] + too_small = [] + for i, r in enumerate(results): + #results return "None" if txo is spent; drop this + if not r: + continue + valid_age = r['confirms'] >= age + valid_amt = r['value'] >= amt + if not valid_age: + too_old.append(utxos[i]) + if not valid_amt: + too_small.append(utxos[i]) + if valid_age and valid_amt: + newresults.append(utxos[i]) + + return newresults, too_old, too_small + + def priv_utxo_pairs_from_utxos(utxos, age, amt): + #returns pairs list of (priv, utxo) for each valid utxo; + #also returns lists "too_old" and "too_small" for any + #utxos that did not satisfy the criteria for debugging. + priv_utxo_pairs = [] + new_utxos, too_old, too_small = filter_by_coin_age_amt(utxos.keys(), + age, amt) + new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos} + for k, v in new_utxos_dict.iteritems(): + addr = v['address'] + priv = self.wallet.get_key_from_addr(addr) + if priv: #can be null from create-unsigned + priv_utxo_pairs.append((priv, k)) + return priv_utxo_pairs, too_old, too_small + + commit_type_byte = "P" + podle_data = None + tries = jm_single().config.getint("POLICY", "taker_utxo_retries") + age = jm_single().config.getint("POLICY", "taker_utxo_age") + #Minor rounding errors don't matter here + amt = int(self.cjamount * + jm_single().config.getint("POLICY", + "taker_utxo_amtpercent") / 100.0) + priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos(self.input_utxos, + age, amt) + #Note that we ignore the "too old" and "too small" lists in the first + #pass through, because the same utxos appear in the whole-wallet check. + + #For podle data format see: podle.PoDLE.reveal() + #In first round try, don't use external commitments + podle_data = generate_podle(priv_utxo_pairs, tries) + if not podle_data: + #We defer to a second round to try *all* utxos in wallet; + #this is because it's much cleaner to use the utxos involved + #in the transaction, about to be consumed, rather than use + #random utxos that will persist after. At this step we also + #allow use of external utxos in the json file. + if self.wallet.unspent: + priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos( + self.wallet.unspent, age, amt) + #Pre-filter the set of external commitments that work for this + #transaction according to its size and age. + dummy, extdict = get_podle_commitments() + if len(extdict.keys()) > 0: + ext_valid, ext_to, ext_ts = filter_by_coin_age_amt( + extdict.keys(), age, amt) + else: + ext_valid = None + podle_data = generate_podle(priv_utxo_pairs, tries, ext_valid) + if podle_data: + jlog.debug("Generated PoDLE: " + pprint.pformat(podle_data)) + revelation = PoDLE(u=podle_data['utxo'], + P=podle_data['P'], + P2=podle_data['P2'], + s=podle_data['sig'], + e=podle_data['e']).serialize_revelation() + return (commit_type_byte + podle_data["commit"], revelation, + "Commitment sourced OK") + else: + #we know that priv_utxo_pairs all passed age and size tests, so + #they must have failed the retries test. Summarize this info, + #return error message to caller, and also dump to commitments_debug.txt + errmsg = "" + errmsgheader = ("Failed to source a commitment; this debugging information" + " may help:\n\n") + errmsg += ("1: Utxos that passed age and size limits, but have " + "been used too many times (see taker_utxo_retries " + "in the config):\n") + if len(priv_utxo_pairs) == 0: + errmsg += ("None\n") + else: + for p, u in priv_utxo_pairs: + errmsg += (str(u) + "\n") + errmsg += ("2: Utxos that have less than " + jm_single( + ).config.get("POLICY", "taker_utxo_age") + " confirmations:\n") + if len(to) == 0: + errmsg += ("None\n") + else: + for t in to: + errmsg += (str(t) + "\n") + errmsg += ("3: Utxos that were not at least " + \ + jm_single().config.get( + "POLICY", "taker_utxo_amtpercent") + "% of the " + "size of the coinjoin amount " + str( + self.cjamount) + "\n") + if len(ts) == 0: + errmsg += ("None\n") + else: + for t in ts: + errmsg += (str(t) + "\n") + errmsg += ('***\n') + errmsg += ("Utxos that appeared in item 1 cannot be used again.\n") + errmsg += ( + "Utxos only in item 2 can be used by waiting for more " + "confirmations, (set by the value of taker_utxo_age).\n") + errmsg += ("Utxos only in item 3 are not big enough for this " + "coinjoin transaction, set by the value " + "of taker_utxo_amtpercent.\n") + errmsg += ( + "If you cannot source a utxo from your wallet according " + "to these rules, use the tool add-utxo.py to source a " + "utxo external to your joinmarket wallet. Read the help " + "with 'python add-utxo.py --help'\n\n") + errmsg += ("You can also reset the rules in the joinmarket.cfg " + "file, but this is generally inadvisable.\n") + errmsg += ( + "***\nFor reference, here are the utxos in your wallet:\n") + errmsg += ("\n" + str(self.wallet.unspent)) + + with open("commitments_debug.txt", "wb") as f: + errmsgfileheader = ("THIS IS A TEMPORARY FILE FOR DEBUGGING; " + "IT CAN BE SAFELY DELETED ANY TIME.\n") + errmsgfileheader += ("***\n") + f.write(errmsgfileheader + errmsg) + + return (None, None, errmsgheader + errmsg) + + def get_commitment(self, utxos, amount): + """Create commitment to fulfil anti-DOS requirement of makers, + storing the corresponding reveal/proof data for next step. + """ + while True: + self.commitment, self.reveal_commitment = self.make_commitment( + self.wallet, utxos, amount) + if (self.commitment) or (jm_single().wait_for_commitments == 0): + break + jlog.debug("Failed to source commitments, waiting 3 minutes") + time.sleep(3 * 60) + if not self.commitment: + jlog.debug( + "Cannot construct transaction, failed to generate " + "commitment, shutting down. Please read commitments_debug.txt " + "for some information on why this is, and what can be " + "done to remedy it.") + #TODO: would like to raw_input here to show the user, but + #interactivity is undesirable here. + #Test only: + if jm_single().config.get("BLOCKCHAIN", + "blockchain_source") == 'regtest': + raise PoDLEError("For testing raising podle exception") + #The timeout/recovery code is designed to handle non-responsive + #counterparties, but this condition means that the current bot + #is not able to create transactions following its *own* rules, + #so shutting down is appropriate no matter what style + #of bot this is. + #These two settings shut down the timeout thread and avoid recovery. + self.all_responded = True + self.end_timeout_thread = True + self.msgchan.shutdown() + + def coinjoin_address(self): + if self.my_cj_addr: + return self.my_cj_addr + else: + addr, self.sign_k = donation_address() + return addr + + def sign_tx(self, tx, i, priv): + if self.my_cj_addr: + return btc.sign(tx, i, priv) + else: + return btc.sign(tx, + i, + priv, + usenonce=btc.safe_hexlify(self.sign_k)) + + def self_sign(self): + # now sign it ourselves + tx = btc.serialize(self.latest_tx) + if self.sign_method == "wallet": + #Currently passes addresses of to-be-signed inputs + #to backend wallet; this is correct for Electrum, may need + #different info for other backends. + addrs = {} + for index, ins in enumerate(self.latest_tx['ins']): + utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + if utxo not in self.input_utxos.keys(): + continue + addrs[index] = self.input_utxos[utxo]['address'] + tx = self.wallet.sign_tx(btc.serialize(wallet_tx), addrs) + else: + for index, ins in enumerate(self.latest_tx['ins']): + utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + if utxo not in self.input_utxos.keys(): + continue + addr = self.input_utxos[utxo]['address'] + tx = self.sign_tx(tx, index, self.wallet.get_key_from_addr(addr)) + self.latest_tx = btc.deserialize(tx) + + def push(self): + tx = btc.serialize(self.latest_tx) + jlog.debug('\n' + tx) + self.txid = btc.txhash(tx) + jlog.debug('txid = ' + self.txid) + pushed = jm_single().bc_interface.pushtx(tx) + return pushed + + def self_sign_and_push(self): + self.self_sign() + return self.push() diff --git a/client/wallet.py b/client/wallet.py new file mode 100644 index 0000000..1931db1 --- /dev/null +++ b/client/wallet.py @@ -0,0 +1,465 @@ +from __future__ import print_function +import json +import os +import pprint +import sys +from decimal import Decimal + +from ConfigParser import NoSectionError +from getpass import getpass + +import btc +from joinmarketclient.slowaes import decryptData +from joinmarketclient.blockchaininterface import BitcoinCoreInterface, RegtestBitcoinCoreInterface +from joinmarketclient.configure import jm_single, get_network, get_p2pk_vbyte + +from joinmarketclient.support import get_log, select_gradual, select_greedy, \ + select_greediest, select + +log = get_log() + +def estimate_tx_fee(ins, outs, txtype='p2pkh'): + '''Returns an estimate of the number of satoshis required + for a transaction with the given number of inputs and outputs, + based on information from the blockchain interface. + ''' + tx_estimated_bytes = btc.estimate_tx_size(ins, outs, txtype) + log.debug("Estimated transaction size: "+str(tx_estimated_bytes)) + fee_per_kb = jm_single().bc_interface.estimate_fee_per_kb( + jm_single().config.getint("POLICY", "tx_fees")) + absurd_fee = jm_single().config.getint("POLICY", "absurd_fee_per_kb") + if fee_per_kb > absurd_fee: + #This error is considered critical; for safety reasons, shut down. + raise ValueError("Estimated fee per kB greater than absurd value: " + \ + str(absurd_fee) + ", quitting.") + log.debug("got estimated tx bytes: "+str(tx_estimated_bytes)) + return int((tx_estimated_bytes * fee_per_kb)/Decimal(1000.0)) + +class AbstractWallet(object): + """ + Abstract wallet for use with JoinMarket + Mostly written with Wallet in mind, the default JoinMarket HD wallet + """ + + def __init__(self): + self.max_mix_depth = 0 + self.unspent = None + self.utxo_selector = select + try: + config = jm_single().config + if config.get("POLICY", "merge_algorithm") == "gradual": + self.utxo_selector = select_gradual + elif config.get("POLICY", "merge_algorithm") == "greedy": + self.utxo_selector = select_greedy + elif config.get("POLICY", "merge_algorithm") == "greediest": + self.utxo_selector = select_greediest + elif config.get("POLICY", "merge_algorithm") != "default": + raise Exception("Unknown merge algorithm") + except NoSectionError: + pass + + def get_key_from_addr(self, addr): + return None + + def get_utxos_by_mixdepth(self): + return None + + def get_external_addr(self, mixing_depth): + """ + Return an address suitable for external distribution, including funding + the wallet from other sources, or receiving payments or donations. + JoinMarket will never generate these addresses for internal use. + """ + return None + + def get_internal_addr(self, mixing_depth): + """ + Return an address for internal usage, as change addresses and when + participating in transactions initiated by other parties. + """ + return None + + def update_cache_index(self): + pass + + def remove_old_utxos(self, tx): + pass + + def add_new_utxos(self, tx, txid): + pass + + def select_utxos(self, mixdepth, amount): + utxo_list = self.get_utxos_by_mixdepth()[mixdepth] + unspent = [{'utxo': utxo, + 'value': addrval['value']} + for utxo, addrval in utxo_list.iteritems()] + inputs = self.utxo_selector(unspent, amount) + log.debug('for mixdepth={} amount={} selected:'.format( + mixdepth, amount)) + log.debug(pprint.pformat(inputs)) + return dict([(i['utxo'], {'value': i['value'], + 'address': utxo_list[i['utxo']]['address']}) + for i in inputs]) + + def get_balance_by_mixdepth(self): + mix_balance = {} + for m in range(self.max_mix_depth): + mix_balance[m] = 0 + for mixdepth, utxos in self.get_utxos_by_mixdepth().iteritems(): + mix_balance[mixdepth] = sum( + [addrval['value'] for addrval in utxos.values()]) + return mix_balance + +class ElectrumWrapWallet(AbstractWallet): + """A thin wrapper class over Electrum's own + wallet for joinmarket compatibility + """ + def __init__(self, ewallet): + self.ewallet = ewallet + #TODO: populate self.unspent with all utxos in Electrum wallet. + + # None is valid for unencrypted electrum wallets; + # calling functions must set the password otherwise + # for private key operations to work + self.password = None + super(ElectrumWrapWallet, self).__init__() + + def get_key_from_addr(self, addr): + if self.ewallet.has_password() and self.password is None: + raise Exception("Cannot extract private key without password") + log.debug("in get key from addr") + log.debug("password is: " + str(self.password)) + log.debug("address is: " + str(addr)) + key = self.ewallet.get_private_key(addr, self.password) + #TODO remove after testing! + log.debug("Got WIF key: " + str(key)) + #Convert from wif compressed to hex compressed + #TODO check if compressed + hex_key = btc.from_wif_privkey(key[0], vbyte=get_p2pk_vbyte()) + log.debug("Got hex key: " + str(hex_key)) + return hex_key + + def get_external_addr(self, mixdepth): + addr = self.ewallet.get_unused_address() + log.debug("Retrieved unused: " + addr) + return addr + + def get_internal_addr(self, mixdepth): + try: + addrs = self.ewallet.get_change_addresses()[ + -self.ewallet.gap_limit_for_change:] + except Exception as e: + log.debug("Failed get change addresses: " + repr(e)) + raise + #filter by unused + try: + change_addrs = [addr for addr in addrs if + self.ewallet.get_num_tx(addr) == 0] + except Exception as e: + log.debug("Failed to filter chadr: " + repr(e)) + raise + #if no unused Electrum re-uses randomly TODO consider + #(of course, all coins in same mixdepth are in principle linkable, + #so I suspect it is better to stick with Electrum's own model, considering + #gap limit issues) + if not change_addrs: + try: + change_addrs = [random.choice(addrs)] + except Exception as e: + log.debug("Failed random: " + repr(e)) + raise + return change_addrs[0] + + def sign_tx(self, tx, addrs): + """tx should be a serialized hex tx. + If self.password is correctly set, + will return the raw transaction with all + inputs from this wallet signed. + """ + if not self.password: + raise Exception("No password, cannot sign") + from electrum.transaction import Transaction + etx = Transaction(tx) + etx.deserialize() + for i in addrs.keys(): + del etx._inputs[i]['scriptSig'] + self.ewallet.add_input_sig_info(etx._inputs[i], addrs[i]) + etx._inputs[i]['address'] = addrs[i] + log.debug("Input is now: " + str(etx._inputs[i])) + self.ewallet.sign_transaction(etx, self.password) + return etx.raw + + def sign_message(self, address, message): + #TODO: not currently used, can we use it for auth? + return self.ewallet.sign_message(address, message, self.password) + + def get_utxos_by_mixdepth(self): + """Initial version: all underlying utxos are mixdepth 0. + Format of return is therefore: {0: + {txid:n : {"address": addr, "value": value}, + txid:n: {"address": addr, "value": value},..}} + TODO this should use the account feature in Electrum, + which is exactly that from BIP32, to implement + multiple mixdepths. + """ + ubym = {0:{}} + coins = self.ewallet.get_spendable_coins() + log.debug(pprint.pformat(coins)) + for c in coins: + utxo = c["prevout_hash"] + ":" + str(c["prevout_n"]) + ubym[0][utxo] = {"address": c["address"], "value": c["value"]} + return ubym + +class Wallet(AbstractWallet): + def __init__(self, + seedarg, + max_mix_depth=2, + gaplimit=6, + extend_mixdepth=False, + storepassword=False): + super(Wallet, self).__init__() + self.max_mix_depth = max_mix_depth + self.storepassword = storepassword + # key is address, value is (mixdepth, forchange, index) if mixdepth = + # -1 it's an imported key and index refers to imported_privkeys + self.addr_cache = {} + self.unspent = {} + self.spent_utxos = [] + self.imported_privkeys = {} + self.seed = self.read_wallet_file_data(seedarg) + if extend_mixdepth and len(self.index_cache) > max_mix_depth: + self.max_mix_depth = len(self.index_cache) + self.gaplimit = gaplimit + master = btc.bip32_master_key(self.seed, (btc.MAINNET_PRIVATE if + get_network() == 'mainnet' else btc.TESTNET_PRIVATE)) + m_0 = btc.bip32_ckd(master, 0) + mixing_depth_keys = [btc.bip32_ckd(m_0, c) + for c in range(self.max_mix_depth)] + self.keys = [(btc.bip32_ckd(m, 0), btc.bip32_ckd(m, 1)) + for m in mixing_depth_keys] + + # self.index = [[0, 0]]*max_mix_depth + self.index = [] + for i in range(self.max_mix_depth): + self.index.append([0, 0]) + + def read_wallet_file_data(self, filename, pwd=None): + self.path = None + self.index_cache = [[0, 0]] * self.max_mix_depth + path = os.path.join('wallets', filename) + if not os.path.isfile(path): + if get_network() == 'testnet': + log.debug('filename interpreted as seed, only available in ' + 'testnet because this probably has lower entropy') + return filename + else: + raise IOError('wallet file not found') + self.path = path + fd = open(path, 'r') + walletfile = fd.read() + fd.close() + walletdata = json.loads(walletfile) + if walletdata['network'] != get_network(): + print ('wallet network(%s) does not match ' + 'joinmarket configured network(%s)' % ( + walletdata['network'], get_network())) + sys.exit(0) + if 'index_cache' in walletdata: + self.index_cache = walletdata['index_cache'] + decrypted = False + while not decrypted: + if pwd: + password = pwd + else: + password = getpass('Enter wallet decryption passphrase: ') + password_key = btc.bin_dbl_sha256(password) + encrypted_seed = walletdata['encrypted_seed'] + try: + decrypted_seed = decryptData( + password_key, + encrypted_seed.decode('hex')).encode('hex') + # there is a small probability of getting a valid PKCS7 + # padding by chance from a wrong password; sanity check the + # seed length + if len(decrypted_seed) == 32: + decrypted = True + else: + raise ValueError + except ValueError: + print('Incorrect password') + if pwd: + raise + decrypted = False + if self.storepassword: + self.password_key = password_key + self.walletdata = walletdata + if 'imported_keys' in walletdata: + for epk_m in walletdata['imported_keys']: + privkey = decryptData( + password_key, + epk_m['encrypted_privkey'].decode( 'hex')).encode('hex') + #Imported keys are stored as 32 byte strings only, so the + #second version below is sufficient, really. + if len(privkey) != 64: + raise Exception( + "Unexpected privkey format; already compressed?:" + privkey) + privkey += "01" + if epk_m['mixdepth'] not in self.imported_privkeys: + self.imported_privkeys[epk_m['mixdepth']] = [] + self.addr_cache[btc.privtoaddr( + privkey, magicbyte=get_p2pk_vbyte())] = (epk_m['mixdepth'], -1, + len(self.imported_privkeys[epk_m['mixdepth']])) + self.imported_privkeys[epk_m['mixdepth']].append(privkey) + return decrypted_seed + + def update_cache_index(self): + if not self.path: + return + if not os.path.isfile(self.path): + return + fd = open(self.path, 'r') + walletfile = fd.read() + fd.close() + walletdata = json.loads(walletfile) + walletdata['index_cache'] = self.index + walletfile = json.dumps(walletdata) + fd = open(self.path, 'w') + fd.write(walletfile) + fd.close() + + def get_key(self, mixing_depth, forchange, i): + return btc.bip32_extract_key(btc.bip32_ckd( + self.keys[mixing_depth][forchange], i)) + + def get_addr(self, mixing_depth, forchange, i): + return btc.privtoaddr( + self.get_key(mixing_depth, forchange, i), magicbyte=get_p2pk_vbyte()) + + def get_new_addr(self, mixing_depth, forchange): + index = self.index[mixing_depth] + addr = self.get_addr(mixing_depth, forchange, index[forchange]) + self.addr_cache[addr] = (mixing_depth, forchange, index[forchange]) + index[forchange] += 1 + # self.update_cache_index() + bc_interface = jm_single().bc_interface + if isinstance(bc_interface, BitcoinCoreInterface) or isinstance( + bc_interface, RegtestBitcoinCoreInterface): + # do not import in the middle of sync_wallet() + if bc_interface.wallet_synced: + if bc_interface.rpc('getaccount', [addr]) == '': + log.debug('importing address ' + addr + ' to bitcoin core') + bc_interface.rpc( + 'importaddress', + [addr, bc_interface.get_wallet_name(self), False]) + return addr + + def get_external_addr(self, mixing_depth): + return self.get_new_addr(mixing_depth, 0) + + def get_internal_addr(self, mixing_depth): + return self.get_new_addr(mixing_depth, 1) + + def get_key_from_addr(self, addr): + if addr not in self.addr_cache: + return None + ac = self.addr_cache[addr] + if ac[1] >= 0: + return self.get_key(*ac) + else: + return self.imported_privkeys[ac[0]][ac[2]] + + def remove_old_utxos(self, tx): + removed_utxos = {} + for ins in tx['ins']: + utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + if utxo not in self.unspent: + continue + removed_utxos[utxo] = self.unspent[utxo] + del self.unspent[utxo] + log.debug('removed utxos, wallet now is \n' + pprint.pformat( + self.get_utxos_by_mixdepth())) + self.spent_utxos += removed_utxos.keys() + return removed_utxos + + def add_new_utxos(self, tx, txid): + added_utxos = {} + for index, outs in enumerate(tx['outs']): + addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) + if addr not in self.addr_cache: + continue + addrdict = {'address': addr, 'value': outs['value']} + utxo = txid + ':' + str(index) + added_utxos[utxo] = addrdict + self.unspent[utxo] = addrdict + log.debug('added utxos, wallet now is \n' + pprint.pformat( + self.get_utxos_by_mixdepth())) + return added_utxos + + def get_utxos_by_mixdepth(self): + """ + returns a list of utxos sorted by different mix levels + """ + mix_utxo_list = {} + for m in range(self.max_mix_depth): + mix_utxo_list[m] = {} + for utxo, addrvalue in self.unspent.iteritems(): + mixdepth = self.addr_cache[addrvalue['address']][0] + if mixdepth not in mix_utxo_list: + mix_utxo_list[mixdepth] = {} + mix_utxo_list[mixdepth][utxo] = addrvalue + log.debug('get_utxos_by_mixdepth = \n' + pprint.pformat(mix_utxo_list)) + return mix_utxo_list + + +class BitcoinCoreWallet(AbstractWallet): + def __init__(self, fromaccount): + super(BitcoinCoreWallet, self).__init__() + if not isinstance(jm_single().bc_interface, + BitcoinCoreInterface): + raise RuntimeError('Bitcoin Core wallet can only be used when ' + 'blockchain interface is BitcoinCoreInterface') + self.fromaccount = fromaccount + self.max_mix_depth = 1 + + def get_key_from_addr(self, addr): + self.ensure_wallet_unlocked() + wifkey = jm_single().bc_interface.rpc('dumpprivkey', [addr]) + return btc.from_wif_privkey(wifkey, vbyte=get_p2pk_vbyte()) + + def get_utxos_by_mixdepth(self): + unspent_list = jm_single().bc_interface.rpc('listunspent', []) + result = {0: {}} + for u in unspent_list: + if not u['spendable']: + continue + if self.fromaccount and ( + ('account' not in u) or u['account'] != + self.fromaccount): + continue + result[0][u['txid'] + ':' + str(u['vout'])] = { + 'address': u['address'], + 'value': int(Decimal(str(u['amount'])) * Decimal('1e8'))} + return result + + def get_internal_addr(self, mixing_depth): + return jm_single().bc_interface.rpc('getrawchangeaddress', []) + + @staticmethod + def ensure_wallet_unlocked(): + wallet_info = jm_single().bc_interface.rpc('getwalletinfo', []) + if 'unlocked_until' in wallet_info and wallet_info[ + 'unlocked_until'] <= 0: + while True: + password = getpass( + 'Enter passphrase to unlock wallet: ') + if password == '': + raise RuntimeError('Aborting wallet unlock') + try: + # TODO cleanly unlock wallet after use, not with arbitrary timeout + jm_single().bc_interface.rpc( + 'walletpassphrase', [password, 10]) + break + except jm_single().JsonRpcError as exc: + if exc.code != -14: + raise exc + # Wrong passphrase, try again. diff --git a/daemon/__init__.py b/daemon/__init__.py new file mode 100644 index 0000000..6812c4b --- /dev/null +++ b/daemon/__init__.py @@ -0,0 +1,22 @@ +from __future__ import print_function + +import logging +from protocol import * +from .enc_wrapper import as_init_encryption, decode_decrypt, \ + encrypt_encode, init_keypair, init_pubkey, get_pubkey, NaclError +from .irc import IRCMessageChannel, B_PER_SEC +from .support import get_log +from .message_channel import MessageChannel, MessageChannelCollection +from .orderbookwatch import OrderbookWatch +import commands + +# Set default logging handler to avoid "No handler found" warnings. +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logging.getLogger(__name__).addHandler(NullHandler()) + diff --git a/daemon/enc_wrapper.py b/daemon/enc_wrapper.py new file mode 100644 index 0000000..10e8ac2 --- /dev/null +++ b/daemon/enc_wrapper.py @@ -0,0 +1,99 @@ +from __future__ import absolute_import, print_function + +# A wrapper for public key +# authenticated encryption +# using Diffie Hellman key +# exchange to set up a +# symmetric encryption. + +import binascii +import base64 +import string +import random + + +from libnacl import public + +class NaclError(Exception): + pass + +def init_keypair(fname=None): + """Create a new encryption + keypair; stored in file fname + if provided. The keypair object + is returned. + """ + kp = public.SecretKey() + if fname: + # Note: handles correct file permissions + kp.save(fname) + return kp + + +# the next two functions are useful +# for exchaging pubkeys with counterparty +def get_pubkey(kp, as_hex=False): + """Given a keypair object, + return its public key, + optionally in hex.""" + if not isinstance(kp, public.SecretKey): + raise NaclError("Object is not a nacl keypair") + return kp.hex_pk() if as_hex else kp.pk + + +def init_pubkey(hexpk, fname=None): + """Create a pubkey object from a + hex formatted string. + Save to file fname if specified. + """ + try: + bin_pk = binascii.unhexlify(hexpk) + except TypeError: + raise NaclError("Invalid hex") + if not len(bin_pk) == 32: + raise NaclError("Public key must be 32 bytes") + pk = public.PublicKey(binascii.unhexlify(hexpk)) + if fname: + pk.save(fname) + return pk + + +def as_init_encryption(kp, c_pk): + """Given an initialised + keypair kp and a counterparty + pubkey c_pk, create a Box + ready for encryption/decryption. + """ + if not isinstance(c_pk, public.PublicKey): + raise NaclError("Object is not a public key") + if not isinstance(kp, public.SecretKey): + raise NaclError("Object is not a nacl keypair") + return public.Box(kp.sk, c_pk) + + +''' +After initialisation, it's possible +to use the box object returned from +as_init_encryption to directly change +from plaintext to ciphertext: + ciphertext = box.encrypt(plaintext) + plaintext = box.decrypt(ciphertext) +Notes: + 1. use binary format for ctext/ptext + 2. Nonce is handled at the implementation layer. +''' + + +# TODO: Sign, verify. At the moment we are using +# bitcoin signatures so it isn't necessary. + + +# encoding for passing over the wire +def encrypt_encode(msg, box): + encrypted = box.encrypt(msg) + return base64.b64encode(encrypted) + + +def decode_decrypt(msg, box): + decoded = base64.b64decode(msg) + return box.decrypt(decoded) diff --git a/daemon/irc.py b/daemon/irc.py new file mode 100644 index 0000000..cc36353 --- /dev/null +++ b/daemon/irc.py @@ -0,0 +1,472 @@ +from __future__ import absolute_import, print_function + +import base64 +import random +import socket +import ssl +import threading +import time +import Queue + + +from joinmarketdaemon.message_channel import MessageChannel +from joinmarketdaemon.support import get_log, chunks +from joinmarketdaemon.socks import socksocket, setdefaultproxy, PROXY_TYPE_SOCKS5 +from joinmarketdaemon.protocol import * +MAX_PRIVMSG_LEN = 450 +PING_INTERVAL = 300 +PING_TIMEOUT = 60 + +#Throttling parameters; data from +#tests by @chris-belcher: +##worked (bytes per sec/bytes per sec interval / counterparties / max_privmsg_len) +#300/4 / 6 / 400 +#600/4 / 6 / 400 +#450/4 / 10 / 400 +#450/4 / 10 / 450 +#525/4 / 10 / 450 +##didnt work +#600/4 / 10 / 450 +#600/4 / 10 / 400 +#2000/2 / 10 / 400 +#450/4 / 10 / 475 +MSG_INTERVAL = 0.001 +B_PER_SEC = 450 +B_PER_SEC_INTERVAL = 4.0 + +def get_config_irc_channel(chan_name, btcnet): + channel = "#" + chan_name + if btcnet == "testnet": + channel += "-test" + return channel + +log = get_log() + +def get_irc_text(line): + return line[line[1:].find(':') + 2:] + + +def get_irc_nick(source): + full_nick = source[1:source.find('!')] + return full_nick[:NICK_MAX_ENCODED+2] + + +class ThrottleThread(threading.Thread): + + def __init__(self, irc): + threading.Thread.__init__(self, name='ThrottleThread') + self.daemon = True + self.irc = irc + self.msg_buffer = [] + + def run(self): + log.debug("starting throttle thread") + last_msg_time = 0 + print_throttle_msg = True + while not self.irc.give_up: + self.irc.lockthrottle.acquire() + while not (self.irc.throttleQ.empty() and self.irc.obQ.empty() + and self.irc.pingQ.empty()): + time.sleep(0.0001) #need to avoid cpu spinning if throttled + try: + pingmsg = self.irc.pingQ.get(block=False) + #ping messages are not counted to throttling totals, + #so send immediately + self.irc.sock.sendall(pingmsg + '\r\n') + continue + except Queue.Empty: + pass + except: + log.warn("failed to send ping message on socket") + break + #First throttling mechanism: no more than 1 line + #per MSG_INTERVAL seconds. + x = time.time() - last_msg_time + if x < MSG_INTERVAL: + continue + #Second throttling mechanism: limited kB/s rate + #over the most recent period. + q = time.time() - B_PER_SEC_INTERVAL + #clean out old messages + self.msg_buffer = [_ for _ in self.msg_buffer if _[1] > q] + bytes_recent = sum(len(i[0]) for i in self.msg_buffer) + if bytes_recent > B_PER_SEC * B_PER_SEC_INTERVAL: + if print_throttle_msg: + log.debug("Throttling triggered, with: "+str( + bytes_recent)+ " bytes in the last "+str( + B_PER_SEC_INTERVAL)+" seconds.") + print_throttle_msg = False + continue + print_throttle_msg = True + try: + throttled_msg = self.irc.throttleQ.get(block=False) + except Queue.Empty: + try: + throttled_msg = self.irc.obQ.get(block=False) + except Queue.Empty: + #this code *should* be unreachable. + continue + try: + self.irc.sock.sendall(throttled_msg+'\r\n') + last_msg_time = time.time() + self.msg_buffer.append((throttled_msg, last_msg_time)) + except: + log.error("failed to send on socket") + try: + self.irc.fd.close() + except: pass + break + self.irc.lockthrottle.wait() + self.irc.lockthrottle.release() + + log.debug("Ended throttling thread.") + +class PingThread(threading.Thread): + + def __init__(self, irc): + threading.Thread.__init__(self, name='PingThread') + self.daemon = True + self.irc = irc + + def run(self): + log.debug('starting ping thread') + while not self.irc.give_up: + time.sleep(PING_INTERVAL) + try: + self.irc.ping_reply = False + # maybe use this to calculate the lag one day + self.irc.lockcond.acquire() + self.irc.send_raw('PING LAG' + str(int(time.time() * 1000))) + self.irc.lockcond.wait(PING_TIMEOUT) + self.irc.lockcond.release() + if not self.irc.ping_reply: + log.warn('irc ping timed out') + try: + self.irc.close() + except: + pass + try: + self.irc.fd.close() + except: + pass + try: + self.irc.sock.shutdown(socket.SHUT_RDWR) + self.irc.sock.close() + except: + pass + except IOError as e: + log.debug('ping thread: ' + repr(e)) + log.debug('ended ping thread') + + +# handle one channel at a time +class IRCMessageChannel(MessageChannel): + # close implies it will attempt to reconnect + def close(self): + try: + self.sock.sendall("QUIT\r\n") + except IOError as e: + log.info('errored while trying to quit: ' + repr(e)) + + def shutdown(self): + self.close() + self.give_up = True + + # Maker callbacks + def _announce_orders(self, orderlist): + """This publishes orders to the pit and to + counterparties. Note that it does *not* use chunking. + So, it tries to optimise space usage thusly: + As many complete orderlines are fit onto one line + as possible, and overflow goes onto another line. + Each list entry in orderlist must have format: + !ordername + + Then, what is published is lines of form: + !ordername !ordername .. + + fitting as many list entries as possible onto one line, + up to the limit of the IRC parameters (see MAX_PRIVMSG_LEN). + + Order announce in private is handled by privmsg/_privmsg + using chunking, no longer using this function. + """ + header = 'PRIVMSG ' + self.channel + ' :' + orderlines = [] + for i, order in enumerate(orderlist): + orderlines.append(order) + line = header + ''.join(orderlines) + ' ~' + if len(line) > MAX_PRIVMSG_LEN or i == len(orderlist) - 1: + if i < len(orderlist) - 1: + line = header + ''.join(orderlines[:-1]) + ' ~' + self.send_raw(line) + orderlines = [orderlines[-1]] + + def _pubmsg(self, message): + line = "PRIVMSG " + self.channel + " :" + message + assert len(line) <= MAX_PRIVMSG_LEN + ob = False + if any([x in line for x in offername_list]): + ob = True + self.send_raw(line, ob) + + def _privmsg(self, nick, cmd, message): + """Send a privmsg to an irc counterparty, + using chunking as appropriate for long messages. + """ + ob = True if cmd in offername_list else False + header = "PRIVMSG " + nick + " :" + max_chunk_len = MAX_PRIVMSG_LEN - len(header) - len(cmd) - 4 + # 1 for command prefix 1 for space 2 for trailer + if len(message) > max_chunk_len: + message_chunks = chunks(message, max_chunk_len) + else: + message_chunks = [message] + for m in message_chunks: + trailer = ' ~' if m == message_chunks[-1] else ' ;' + if m == message_chunks[0]: + m = COMMAND_PREFIX + cmd + ' ' + m + self.send_raw(header + m + trailer, ob) + + def change_nick(self, new_nick): + self.nick = new_nick + self.send_raw('NICK ' + self.nick) + + def send_raw(self, line, ob=False): + # Messages are queued and prioritised. + # This is an addressing of github #300 + if line.startswith("PING") or line.startswith("PONG"): + self.pingQ.put(line) + elif ob: + self.obQ.put(line) + else: + self.throttleQ.put(line) + self.lockthrottle.acquire() + self.lockthrottle.notify() + self.lockthrottle.release() + + def __handle_privmsg(self, source, target, message): + nick = get_irc_nick(source) + #ensure return value 'parsed' is length > 2 + if len(message) < 4: + return + if target == self.nick: + if message[0] == '\x01': + endindex = message[1:].find('\x01') + if endindex == -1: + return + ctcp = message[1:endindex + 1] + if ctcp.upper() == 'VERSION': + self.send_raw('PRIVMSG ' + nick + + ' :\x01VERSION xchat 2.8.8 Ubuntu\x01') + return + + if nick not in self.built_privmsg: + self.built_privmsg[nick] = message[:-2] + else: + self.built_privmsg[nick] += message[:-2] + if message[-1] == '~': + parsed = self.built_privmsg[nick] + # wipe the message buffer waiting for the next one + del self.built_privmsg[nick] + log.debug("< : %s' % (nick, errormsg)) + self.privmsg(nick, 'error', errormsg) + raise CJPeerError() + + def pubmsg(self, message): + log.debug('>>pubmsg ' + message) + #Currently there is no joinmarket protocol logic here; + #just pass-through. + self._pubmsg(message) + + def privmsg(self, nick, cmd, message): + log.debug('>>privmsg on %s: ' % (self.hostid) + 'nick=' + nick + ' cmd=' + + cmd + ' msg=' + message) + #forward to the implementation class (use single _ for polymrphsm to work) + self._privmsg(nick, cmd, message) + + def on_pubmsg(self, nick, message): + #Even illegal messages mark a nick as "seen" + if self.on_pubmsg_trigger: + self.on_pubmsg_trigger(nick, self) + if message[0] != COMMAND_PREFIX: + return + commands = message[1:].split(COMMAND_PREFIX) + #DOS vector: repeated !orderbook requests, see #298. + if commands.count('orderbook') > 1: + return + for command in commands: + _chunks = command.split(" ") + if self.check_for_orders(nick, _chunks): + pass + if self.check_for_commitments(nick, _chunks): + pass + elif _chunks[0] == 'cancel': + # !cancel [oid] + try: + oid = int(_chunks[1]) + if self.on_order_cancel: + self.on_order_cancel(nick, oid) + except (ValueError, IndexError) as e: + log.debug("!cancel " + repr(e)) + return + elif _chunks[0] == 'orderbook': + if self.on_orderbook_requested: + self.on_orderbook_requested(nick, self) + else: + # TODO this is for testing/debugging, should be removed, see taker.py + if hasattr(self, 'debug_on_pubmsg_cmd'): + self.debug_on_pubmsg_cmd(nick, _chunks) + + def on_privmsg(self, nick, message): + """handles the case when a private message is received""" + #Aberrant short messages should be handled by subclasses + #in _privmsg, but this constitutes a sanity check. Note that + #messages which use an encrypted_command but present no + #ciphertext will be rejected with the ValueError on decryption. + #Other ill formatted messages will be caught in the try block. + if len(message) < 2: + return + + if message[0] != COMMAND_PREFIX: + log.debug('message not a cmd') + return + cmd_string = message[1:].split(' ')[0] + if cmd_string not in plaintext_commands + encrypted_commands: + log.debug('cmd not in cmd_list, line="' + message + '"') + return + #Verify nick ownership + sig = message[1:].split(' ')[-2:] + #reconstruct original message without cmd + rawmessage = ' '.join(message[1:].split(' ')[1:-2]) + #sanity check that the sig was appended properly + if len(sig) != 2 or len(rawmessage) == 0: + log.debug("Sig not properly appended to privmsg, ignoring") + return + self.daemon.request_signature_verify( + rawmessage + str(self.hostid), message, sig[1], sig[0], nick, + NICK_HASH_LENGTH, NICK_MAX_ENCODED, str(self.hostid)) + + def on_verified_privmsg(self, nick, message): + #Marks the nick as active on this channel; note *only* if verified. + #Otherwise squatter/attacker can persuade us to send privmsgs to him. + if self.on_privmsg_trigger: + self.on_privmsg_trigger(nick, self) + #strip sig from message for processing, having verified + message = " ".join(message[1:].split(" ")[:-2]) + for command in message.split(COMMAND_PREFIX): + _chunks = command.split(" ") + + #Decrypt if necessary + if _chunks[0] in encrypted_commands: + box, encrypt = self.daemon.mcc.get_encryption_box(_chunks[0], + nick) + if encrypt: + if not box: + log.debug('error, dont have encryption box object for ' + + nick + ', dropping message') + return + # need to decrypt everything after the command string + to_decrypt = ''.join(_chunks[1:]) + try: + decrypted = decode_decrypt(to_decrypt, box) + except ValueError as e: + log.debug('valueerror when decrypting, skipping: ' + + repr(e)) + return + #rebuild the chunks array as if it had been plaintext + _chunks = [_chunks[0]] + decrypted.split(" ") + + # looks like a very similar pattern for all of these + # check for a command name, parse arguments, call a function + # maybe we need some eval() trickery to do it better + + try: + # orderbook watch commands + if self.check_for_orders(nick, _chunks): + pass + # taker commands + elif _chunks[0] == 'pubkey': + maker_pk = _chunks[1] + if self.on_pubkey: + self.on_pubkey(nick, maker_pk) + elif _chunks[0] == 'ioauth': + utxo_list = _chunks[1].split(',') + auth_pub = _chunks[2] + cj_addr = _chunks[3] + change_addr = _chunks[4] + btc_sig = _chunks[5] + if self.on_ioauth: + self.on_ioauth(nick, utxo_list, auth_pub, cj_addr, + change_addr, btc_sig) + elif _chunks[0] == 'sig': + sig = _chunks[1] + if self.on_sig: + self.on_sig(nick, sig) + + # maker commands + if self.check_for_commitments(nick, _chunks, private=True): + pass + if _chunks[0] == 'fill': + try: + oid = int(_chunks[1]) + amount = int(_chunks[2]) + taker_pk = _chunks[3] + if len(_chunks) > 4: + commit = _chunks[4] + else: + commit = None + except (ValueError, IndexError) as e: + self.send_error(nick, str(e)) + if self.on_order_fill: + self.on_order_fill(nick, oid, amount, taker_pk, commit) + elif _chunks[0] == 'auth': + try: + cr = _chunks[1] + except (ValueError, IndexError) as e: + self.send_error(nick, str(e)) + if self.on_seen_auth: + self.on_seen_auth(nick, cr) + elif _chunks[0] == 'tx': + b64tx = _chunks[1] + try: + txhex = base64.b64decode(b64tx).encode('hex') + except TypeError as e: + self.send_error(nick, 'bad base64 tx. ' + repr(e)) + if self.on_seen_tx: + self.on_seen_tx(nick, txhex) + elif _chunks[0] == 'push': + b64tx = _chunks[1] + try: + txhex = base64.b64decode(b64tx).encode('hex') + except TypeError as e: + self.send_error(nick, 'bad base64 tx. ' + repr(e)) + if self.on_push_tx: + self.on_push_tx(nick, txhex) + except CJPeerError: + # TODO proper error handling + log.debug('cj peer error TODO handle') + continue diff --git a/daemon/orderbookwatch.py b/daemon/orderbookwatch.py new file mode 100644 index 0000000..edee599 --- /dev/null +++ b/daemon/orderbookwatch.py @@ -0,0 +1,132 @@ +#! /usr/bin/env python +from __future__ import absolute_import, print_function + +import base64 +import pprint +import random +import sqlite3 +import sys +import time +import threading +import json +from decimal import InvalidOperation, Decimal + +from joinmarketdaemon.protocol import JM_VERSION +from joinmarketdaemon.support import get_log, joinmarket_alert, DUST_THRESHOLD +from joinmarketdaemon.irc import B_PER_SEC +log = get_log() + + +class JMTakerError(Exception): + pass + + +class OrderbookWatch(object): + + def set_msgchan(self, msgchan): + self.msgchan = msgchan + self.msgchan.register_orderbookwatch_callbacks(self.on_order_seen, + self.on_order_cancel) + self.msgchan.register_channel_callbacks( + self.on_welcome, self.on_set_topic, None, self.on_disconnect, + self.on_nick_leave, None) + + self.dblock = threading.Lock() + con = sqlite3.connect(":memory:", check_same_thread=False) + con.row_factory = sqlite3.Row + self.db = con.cursor() + self.db.execute("CREATE TABLE orderbook(counterparty TEXT, " + "oid INTEGER, ordertype TEXT, minsize INTEGER, " + "maxsize INTEGER, txfee INTEGER, cjfee TEXT);") + + @staticmethod + def on_set_topic(newtopic): + chunks = newtopic.split('|') + for msg in chunks[1:]: + try: + msg = msg.strip() + params = msg.split(' ') + min_version = int(params[0]) + max_version = int(params[1]) + alert = msg[msg.index(params[1]) + len(params[1]):].strip() + except ValueError, IndexError: + continue + if min_version < JM_VERSION < max_version: + print('=' * 60) + print('JOINMARKET ALERT') + print(alert) + print('=' * 60) + joinmarket_alert[0] = alert + + def on_order_seen(self, counterparty, oid, ordertype, minsize, maxsize, + txfee, cjfee): + try: + self.dblock.acquire(True) + if int(oid) < 0 or int(oid) > sys.maxint: + log.debug("Got invalid order ID: " + oid + " from " + + counterparty) + return (False, []) + # delete orders eagerly, so in case a buggy maker sends an + # invalid offer, we won't accidentally !fill based on the ghost + # of its previous message. + self.db.execute( + ("DELETE FROM orderbook WHERE counterparty=? " + "AND oid=?;"), (counterparty, oid)) + # now validate the remaining fields + if int(minsize) < 0 or int(minsize) > 21 * 10**14: + log.debug("Got invalid minsize: {} from {}".format( + minsize, counterparty)) + return (False, []) + if int(minsize) < DUST_THRESHOLD: + minsize = DUST_THRESHOLD + log.debug("{} has dusty minsize, capping at {}".format( + counterparty, minsize)) + # do not pass return, go not drop this otherwise fine offer + if int(maxsize) < 0 or int(maxsize) > 21 * 10**14: + log.debug("Got invalid maxsize: " + maxsize + " from " + + counterparty) + return (False, []) + if int(txfee) < 0: + log.debug("Got invalid txfee: {} from {}".format(txfee, + counterparty)) + return (False, []) + if int(minsize) > int(maxsize): + + fmt = ("Got minsize bigger than maxsize: {} - {} " + "from {}").format + log.debug(fmt(minsize, maxsize, counterparty)) + return (False, []) + if ordertype == 'absoffer' and not isinstance(cjfee, int): + try: + cjfee = int(cjfee) + except ValueError: + log.debug("Got non integer coinjoin fee: " + str(cjfee) + + " for an absoffer from " + counterparty) + return (False, []) + self.db.execute( + 'INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);', + (counterparty, oid, ordertype, minsize, maxsize, txfee, + str(Decimal(cjfee)))) # any parseable Decimal is a valid cjfee + except InvalidOperation: + log.debug("Got invalid cjfee: " + cjfee + " from " + counterparty) + except Exception as e: + log.debug("Error parsing order " + oid + " from " + counterparty) + log.debug("Exception was: " + repr(e)) + finally: + self.dblock.release() + return (True, []) + + def on_order_cancel(self, counterparty, oid): + with self.dblock: + self.db.execute( + ("DELETE FROM orderbook WHERE " + "counterparty=? AND oid=?;"), (counterparty, oid)) + + def on_nick_leave(self, nick): + with self.dblock: + self.db.execute('DELETE FROM orderbook WHERE counterparty=?;', + (nick,)) + + def on_disconnect(self): + with self.dblock: + self.db.execute('DELETE FROM orderbook;') diff --git a/daemon/protocol.py b/daemon/protocol.py new file mode 100644 index 0000000..e94cf14 --- /dev/null +++ b/daemon/protocol.py @@ -0,0 +1,31 @@ +#Protocol version +JM_VERSION = 5 + +#Username on all messagechannels; will be set in MessageChannelCollection +nickname = None + +separator = " " +offertypes = {"reloffer": [(int, "oid"), (int, "minsize"), (int, "maxsize"), + (int, "txfee"), (float, "cjfee")], + "absoffer": [(int, "oid"), (int, "minsize"), (int, "maxsize"), + (int, "txfee"), (int, "cjfee")]} + +offername_list = offertypes.keys() + +ORDER_KEYS = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', + 'cjfee'] + +COMMAND_PREFIX = '!' +JOINMARKET_NICK_HEADER = 'J' +NICK_HASH_LENGTH = 10 +NICK_MAX_ENCODED = 14 #comes from base58 expansion; recalculate if above changes + +#Lists of valid commands +encrypted_commands = ["auth", "ioauth", "tx", "sig"] +plaintext_commands = ["fill", "error", "pubkey", "orderbook", "push"] +commitment_broadcast_list = ["hp2"] +plaintext_commands += offername_list +plaintext_commands += commitment_broadcast_list +public_commands = commitment_broadcast_list + ["orderbook", "cancel" + ] + offername_list +private_commands = encrypted_commands + plaintext_commands diff --git a/daemon/socks.py b/daemon/socks.py new file mode 100644 index 0000000..af32842 --- /dev/null +++ b/daemon/socks.py @@ -0,0 +1,410 @@ +"""SocksiPy - Python SOCKS module. +Version 1.00 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +""" + +import socket +import struct +import random + +PROXY_TYPE_SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = 2 +PROXY_TYPE_HTTP = 3 + +_defaultproxy = None +_orgsocket = socket.socket + + +class ProxyError(IOError): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +class GeneralProxyError(ProxyError): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +class Socks5AuthError(ProxyError): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +class Socks5Error(ProxyError): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +class Socks4Error(ProxyError): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +class HTTPError(ProxyError): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +_generalerrors = ("success", "invalid data", "not connected", "not available", + "bad proxy type", "bad input") + +_socks5errors = ("succeeded", "general SOCKS server failure", + "connection not allowed by ruleset", "Network unreachable", + "Host unreachable", "Connection refused", "TTL expired", + "Command not supported", "Address type not supported", + "Unknown error") + +_socks5autherrors = ("succeeded", "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", "unknown error") + +_socks4errors = ( + "request granted", "request rejected or failed", + "request rejected because SOCKS server cannot connect to identd on the client", + "request rejected because the client program and identd report different user-ids", + "unknown error") + + +def setdefaultproxy(proxytype=None, + addr=None, + port=None, + rdns=True, + username=str(random.randrange(10000000, 99999999)), + password=str(random.randrange(10000000, 99999999))): + """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. + """ + global _defaultproxy + _defaultproxy = (proxytype, addr, port, rdns, username, password) + + +class socksocket(socket.socket): + """socksocket([family[, type[, proto]]]) -> socket object + + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET, type=SOCK_STREAM and proto=0. + """ + + def __init__(self, + family=socket.AF_INET, + type=socket.SOCK_STREAM, + proto=0, + _sock=None): + _orgsocket.__init__(self, family, type, proto, _sock) + if _defaultproxy is not None: + self.__proxy = _defaultproxy + else: + self.__proxy = (None, None, None, None, None, None) + self.__proxysockname = None + self.__proxypeername = None + + def __recvall(self, bytes): + """__recvall(bytes) -> data + Receive EXACTLY the number of bytes requested from the socket. + Blocks until the required number of bytes have been received. + """ + data = "" + while len(data) < bytes: + data = data + self.recv(bytes - len(data)) + return data + + def setproxy(self, + proxytype=None, + addr=None, + port=None, + rdns=True, + username=None, + password=None): + """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + proxytype - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be preformed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.__proxy = (proxytype, addr, port, rdns, username, password) + + def __negotiatesocks5(self, destaddr, destport): + """__negotiatesocks5(self,destaddr,destport) + Negotiates a connection through a SOCKS5 server. + """ + # First we'll send the authentication packages we support. + if (self.__proxy[4] is not None) and (self.__proxy[5] is not None): + # The username/password details were supplied to the + # setproxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + self.sendall("\x05\x02\x00\x02") + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + self.sendall("\x05\x01\x00") + # We'll receive the server's response to determine which + # method was selected + chosenauth = self.__recvall(2) + if chosenauth[0] != "\x05": + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + # Check the chosen authentication method + if chosenauth[1] == "\x00": + # No authentication is required + pass + elif chosenauth[1] == "\x02": + # Okay, we need to perform a basic username/password + # authentication. + self.sendall("\x01" + chr(len(self.__proxy[4])) + self.__proxy[4] + + chr(len(self.__proxy[5])) + self.__proxy[5]) + authstat = self.__recvall(2) + if authstat[0] != "\x01": + # Bad response + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if authstat[1] != "\x00": + # Authentication failed + self.close() + raise Socks5AuthError, (3, _socks5autherrors[3]) + # Authentication succeeded + else: + # Reaching here is always bad + self.close() + if chosenauth[1] == "\xFF": + raise Socks5AuthError((2, _socks5autherrors[2])) + else: + raise GeneralProxyError((1, _generalerrors[1])) + # Now we can request the actual connection + req = "\x05\x01\x00" + # If the given destination address is an IP address, we'll + # use the IPv4 address request even if remote resolving was specified. + try: + ipaddr = socket.inet_aton(destaddr) + req = req + "\x01" + ipaddr + except socket.error: + # Well it's not an IP number, so it's probably a DNS name. + if self.__proxy[3]: + # Resolve remotely + ipaddr = None + req = req + "\x03" + chr(len(destaddr)) + destaddr + else: + # Resolve locally + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + req = req + "\x01" + ipaddr + req += struct.pack(">H", destport) + self.sendall(req) + # Get the response + resp = self.__recvall(4) + if resp[0] != "\x05": + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + elif resp[1] != "\x00": + # Connection failed + self.close() + raise Socks5Error(_socks5errors[min(9, ord(resp[1]))]) + # Get the bound address/port + elif resp[3] == "\x01": + boundaddr = self.__recvall(4) + elif resp[3] == "\x03": + resp = resp + self.recv(1) + boundaddr = self.__recvall(resp[4]) + else: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + boundport = struct.unpack(">H", self.__recvall(2))[0] + self.__proxysockname = (boundaddr, boundport) + if ipaddr is not None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def getproxysockname(self): + """getsockname() -> address info + Returns the bound IP address and port number at the proxy. + """ + return self.__proxysockname + + def getproxypeername(self): + """getproxypeername() -> address info + Returns the IP and port number of the proxy. + """ + return _orgsocket.getpeername(self) + + def getpeername(self): + """getpeername() -> address info + Returns the IP address and port number of the destination + machine (note: getproxypeername returns the proxy) + """ + return self.__proxypeername + + def __negotiatesocks4(self, destaddr, destport): + """__negotiatesocks4(self,destaddr,destport) + Negotiates a connection through a SOCKS4 server. + """ + # Check if the destination address provided is an IP address + rmtrslv = False + try: + ipaddr = socket.inet_aton(destaddr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if self.__proxy[3]: + ipaddr = "\x00\x00\x00\x01" + rmtrslv = True + else: + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + # Construct the request packet + req = "\x04\x01" + struct.pack(">H", destport) + ipaddr + # The username parameter is considered userid for SOCKS4 + if self.__proxy[4] is not None: + req = req + self.__proxy[4] + req += "\x00" + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if rmtrslv: + req = req + destaddr + "\x00" + self.sendall(req) + # Get the response from the server + resp = self.__recvall(8) + if resp[0] != "\x00": + # Bad data + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if resp[1] != "\x5A": + # Server returned an error + self.close() + if ord(resp[1]) in (91, 92, 93): + self.close() + raise Socks4Error((ord(resp[1]), _socks4errors[ord(resp[1]) - + 90])) + else: + raise Socks4Error((94, _socks4errors[4])) + # Get the bound address/port + self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack( + ">H", resp[2:4])[0]) + if rmtrslv is not None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def __negotiatehttp(self, destaddr, destport): + """__negotiatehttp(self,destaddr,destport) + Negotiates a connection through an HTTP server. + """ + # If we need to resolve locally, we do this now + if not self.__proxy[3]: + addr = socket.gethostbyname(destaddr) + else: + addr = destaddr + self.sendall("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + + "Host: " + destaddr + "\r\n\r\n") + # We read the response until we get the string "\r\n\r\n" + resp = self.recv(1) + while resp.find("\r\n\r\n") == -1: + resp = resp + self.recv(1) + # We just need the first line to check if the connection + # was successful + statusline = resp.splitlines()[0].split(" ", 2) + if statusline[0] not in ("HTTP/1.0", "HTTP/1.1"): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + try: + statuscode = int(statusline[1]) + except ValueError: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if statuscode != 200: + self.close() + raise HTTPError((statuscode, statusline[2])) + self.__proxysockname = ("0.0.0.0", 0) + self.__proxypeername = (addr, destport) + + def connect(self, destpair): + """connect(self,despair) + Connects to the specified destination through a proxy. + destpar - A tuple of the IP/DNS address and the port number. + (identical to socket's connect). + To select the proxy server use setproxy(). + """ + # Do a minimal input check first + if (type(destpair) in + (list, tuple) == False) or (len(destpair) < 2) or ( + type(destpair[0]) != str) or (type(destpair[1]) != int): + raise GeneralProxyError((5, _generalerrors[5])) + if self.__proxy[0] == PROXY_TYPE_SOCKS5: + if self.__proxy[2] is not None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatesocks5(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_SOCKS4: + if self.__proxy[2] is not None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatesocks4(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP: + if self.__proxy[2] is not None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatehttp(destpair[0], destpair[1]) + elif self.__proxy[0] is None: + _orgsocket.connect(self, (destpair[0], destpair[1])) + else: + raise GeneralProxyError((4, _generalerrors[4])) diff --git a/joinmarketd.py b/joinmarketd.py new file mode 100644 index 0000000..c03632e --- /dev/null +++ b/joinmarketd.py @@ -0,0 +1,318 @@ +#! /usr/bin/env python +from __future__ import print_function +import sys +from joinmarketdaemon import (IRCMessageChannel, MessageChannelCollection, + OrderbookWatch, as_init_encryption, init_pubkey, + NaclError, init_keypair, COMMAND_PREFIX, ORDER_KEYS, + NICK_HASH_LENGTH, NICK_MAX_ENCODED, JM_VERSION, + JOINMARKET_NICK_HEADER) + +from joinmarketdaemon.commands import * +from twisted.protocols import amp +from twisted.internet import reactor +from twisted.internet.protocol import ServerFactory +from twisted.python.log import startLogging, err +from twisted.python import log +import json +import time +import threading + + +"""Joinmarket application protocol control flow. +For documentation on protocol (formats, message sequence) see +https://github.com/JoinMarket-Org/JoinMarket-Docs/blob/master/ +Joinmarket-messaging-protocol.md +""" +""" +*** +API +*** +The client-daemon two-way communication is documented in commands.py +""" + + +class MCThread(threading.Thread): + + def __init__(self, mc): + threading.Thread.__init__(self, name='MCThread') + self.mc = mc + self.daemon = True + + def run(self): + self.mc.run() + + +class JMProtocolError(Exception): + pass + +class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): + + def __init__(self, factory): + self.factory = factory + #Set of messages we can receive from a client: + self.supported_messages = ["JM_INIT", "JM_SETUP", "JM_FILL", + "JM_MAKE_TX", "JM_REQUEST_OFFERS", + "JM_MAKE_TX", "JM_MSGSIGNATURE", + "JM_MSGSIGNATURE_VERIFY", "JM_START_MC"] + self.jm_state = 0 + + def checkClientResponse(self, response): + """A generic check of client acceptance; any failure + is considered criticial. + """ + if 'accepted' not in response or not response['accepted']: + reactor.stop() + + @JMInit.responder + def on_JM_INIT(self, bcsource, network, irc_configs, minmakers, + maker_timeout_sec): + self.maker_timeout_sec = int(maker_timeout_sec) + self.minmakers = int(minmakers) + irc_configs = json.loads(irc_configs) + mcs = [IRCMessageChannel(c, + daemon=self, + realname='btcint=' + bcsource) + for c in irc_configs] + #(bitcoin) network only referenced in channel name construction + self.network = network + self.mcc = MessageChannelCollection(mcs) + OrderbookWatch.set_msgchan(self, self.mcc) + #register taker-specific msgchan callbacks here + self.mcc.register_taker_callbacks(self.on_error, self.on_pubkey, + self.on_ioauth, self.on_sig) + self.mcc.set_daemon(self) + d = self.callRemote(JMInitProto, + nick_hash_length=NICK_HASH_LENGTH, + nick_max_encoded=NICK_MAX_ENCODED, + joinmarket_nick_header=JOINMARKET_NICK_HEADER, + joinmarket_version=JM_VERSION) + d.addCallback(self.checkClientResponse) + return {'accepted': True} + + @JMStartMC.responder + def on_JM_START_MC(self, nick): + """Starts message channel threads; + JM_UP will be called when the welcome messages are received. + """ + self.init_connections(nick) + return {'accepted': True} + + def init_connections(self, nick): + self.jm_state = 0 #uninited + self.mcc.set_nick(nick) + MCThread(self.mcc).start() + + def on_welcome(self): + """Fired when channel indicated state readiness + """ + d = self.callRemote(JMUp) + d.addCallback(self.checkClientResponse) + + @JMSetup.responder + def on_JM_SETUP(self, role, n_counterparties): + assert self.jm_state == 0 + assert n_counterparties > 1 + #TODO consider MAKER role implementation here + assert role == "TAKER" + self.requested_counterparties = n_counterparties + self.crypto_boxes = {} + self.kp = init_keypair() + print("Received setup command") + d = self.callRemote(JMSetupDone) + d.addCallback(self.checkClientResponse) + #Request orderbook here, on explicit setup request from client, + #assumes messagechannels are in "up" state. Orders are read + #in the callback on_order_seen in OrderbookWatch. + self.mcc.pubmsg(COMMAND_PREFIX + "orderbook") + self.jm_state = 1 + return {'accepted': True} + + @JMRequestOffers.responder + def on_JM_REQUEST_OFFERS(self): + """Reports the current state of the orderbook. + This call is stateless.""" + rows = self.db.execute('SELECT * FROM orderbook;').fetchall() + self.orderbook = [dict([(k, o[k]) for k in ORDER_KEYS]) for o in rows] + log.msg("About to send orderbook of size: " + str(len(self.orderbook))) + string_orderbook = json.dumps(self.orderbook) + d = self.callRemote(JMOffers, + orderbook=string_orderbook) + d.addCallback(self.checkClientResponse) + return {'accepted': True} + + @JMFill.responder + def on_JM_FILL(self, amount, commitment, revelation, filled_offers): + if not (self.jm_state == 1 and isinstance(amount, int) and amount >=0): + return {'accepted': False} + self.cjamount = amount + self.commitment = commitment + self.revelation = revelation + #Reset utxo data to null for this new transaction + self.ioauth_data = {} + self.active_orders = json.loads(filled_offers) + for nick, offer_dict in self.active_orders.iteritems(): + offer_fill_msg = " ".join([str(offer_dict["oid"]), str(amount), str( + self.kp.hex_pk()), str(commitment)]) + self.mcc.prepare_privmsg(nick, "fill", offer_fill_msg) + self.first_stage_timer = time.time() + self.jm_state = 2 + return {'accepted': True} + + def on_pubkey(self, nick, maker_pk): + """This is handled locally in the daemon; set up e2e + encrypted messaging with this counterparty + """ + if nick not in self.active_orders.keys(): + log.msg("Counterparty not part of this transaction. Ignoring") + return + try: + self.crypto_boxes[nick] = [maker_pk, as_init_encryption( + self.kp, init_pubkey(maker_pk))] + except NaclError as e: + print("Unable to setup crypto box with " + nick + ": " + repr(e)) + self.mcc.send_error(nick, "invalid nacl pubkey: " + maker_pk) + return + self.mcc.prepare_privmsg(nick, "auth", str(self.revelation)) + + def on_ioauth(self, nick, utxo_list, auth_pub, cj_addr, change_addr, + btc_sig): + """Passes through to Taker the information from counterparties once + they've all been received; note that we must also pass back the maker_pk + so it can be verified against the btc-sigs for anti-MITM + """ + def respond(accepted): + d = self.callRemote(JMFillResponse, + success=accepted, + ioauth_data = json.dumps(self.ioauth_data)) + if not accepted: + #Client simply accepts failure TODO + d.addCallback(self.checkClientResponse) + else: + #Act differently if *we* provided utxos, but + #client does not accept for some reason + d.addCallback(self.checkUtxosAccepted) + + if nick not in self.active_orders.keys(): + print("Got an unexpected ioauth from nick: " + str(nick)) + return + self.ioauth_data[nick] = [utxo_list, auth_pub, cj_addr, change_addr, + btc_sig, self.crypto_boxes[nick][0]] + if self.ioauth_data.keys() == self.active_orders.keys(): + respond(True) + else: + time_taken = time.time() - self.first_stage_timer + #if the timer has run out, either pass through if we have + #at least minmakers, else return a failure condition + if time_taken > self.maker_timeout_sec: + if len(self.ioauth_data.keys()) >= self.minmakers: + respond(True) + else: + respond(False) + + def checkUtxosAccepted(self, accepted): + if not accepted: + log.msg("Taker rejected utxos provided; resetting.") + #TODO create re-set function to start again + + @JMMakeTx.responder + def on_JM_MAKE_TX(self, nick_list, txhex): + if not self.jm_state == 2: + return {'accepted': False} + nick_list = json.loads(nick_list) + self.mcc.send_tx(nick_list, txhex) + return {'accepted': True} + + def on_sig(self, nick, sig): + """Pass signature through to Taker. + """ + d = self.callRemote(JMSigReceived, + nick=nick, + sig=sig) + d.addCallback(self.checkClientResponse) + + """The following functions handle requests and responses + from client for messaging signing and verifying. + """ + def request_signed_message(self, nick, cmd, msg, msg_to_be_signed, hostid): + """The daemon passes the nick and cmd fields + to the client so it can be echoed back to the privmsg + after return (with signature); note that the cmd is already + inside "msg" after having been parsed in MessageChannel; this + duplication is so that the client does not need to know the + message syntax. + """ + d = self.callRemote(JMRequestMsgSig, + nick=str(nick), + cmd=str(cmd), + msg=str(msg), + msg_to_be_signed=str(msg_to_be_signed), + hostid=str(hostid)) + d.addCallback(self.checkClientResponse) + + def request_signature_verify(self, msg, fullmsg, sig, pubkey, nick, hashlen, + max_encoded, hostid): + d = self.callRemote(JMRequestMsgSigVerify, + msg=msg, + fullmsg=fullmsg, + sig=sig, + pubkey=pubkey, + nick=nick, + hashlen=hashlen, + max_encoded=max_encoded, + hostid=hostid) + d.addCallback(self.checkClientResponse) + + @JMMsgSignature.responder + def on_JM_MSGSIGNATURE(self, nick, cmd, msg_to_return, hostid): + self.mcc.privmsg(nick, cmd, msg_to_return, mc=hostid) + return {'accepted': True} + + @JMMsgSignatureVerify.responder + def on_JM_MSGSIGNATURE_VERIFY(self, verif_result, nick, fullmsg, hostid): + if not verif_result: + log.msg("Verification failed for nick: " + str(nick)) + else: + self.mcc.on_verified_privmsg(nick, fullmsg, hostid) + return {'accepted': True} + + def get_crypto_box_from_nick(self, nick): + if nick in self.crypto_boxes and self.crypto_boxes[nick] != None: + return self.crypto_boxes[nick][1] # libsodium encryption object + else: + log.msg('something wrong, no crypto object, nick=' + nick + + ', message will be dropped') + return None + + def on_error(self): + log.msg("Unimplemented on_error") + + def mc_shutdown(self): + log.msg("Message channels shut down in proto") + self.mcc.shutdown() + + +class JMDaemonServerProtocolFactory(ServerFactory): + protocol = JMDaemonServerProtocol + + def buildProtocol(self, addr): + return JMDaemonServerProtocol(self) + +def startup_joinmarketd(port, finalizer=None, finalizer_args=None): + """Start event loop for joinmarket daemon here. + Args: + port : port over which to serve the daemon + finalizer: a function which is called after the reactor has shut down. + finalizer_args : arguments to finalizer function. + """ + log.startLogging(sys.stdout) + factory = JMDaemonServerProtocolFactory() + reactor.listenTCP(port, factory) + if finalizer: + reactor.addSystemEventTrigger("after", "shutdown", finalizer, + finalizer_args) + reactor.run() + + +if __name__ == "__main__": + port = int(sys.argv[1]) + startup_joinmarketd(port) diff --git a/sendpayment.py b/sendpayment.py new file mode 100644 index 0000000..a812043 --- /dev/null +++ b/sendpayment.py @@ -0,0 +1,201 @@ +#! /usr/bin/env python +from __future__ import absolute_import, print_function + +""" +A sample implementation of a single coinjoin script, +adapted from `sendpayment.py` in Joinmarket-Org/joinmarket. +More complex applications can extend from Taker and add +more features, such as repeated joins. This will also allow +easier coding of non-CLI interfaces. + +Other potential customisations of the Taker object instantiation +include: + +external_addr=None implies joining to another mixdepth +in the same wallet. + +order_chooser can be set to a different custom function that selects +counterparty offers according to different rules. +""" + +import random +import sys +import threading +from optparse import OptionParser + +import time + +from joinmarketclient import (Taker, load_program_config, + JMTakerClientProtocolFactory, start_reactor, + validate_address, jm_single, get_log, + choose_orders, choose_sweep_orders, pick_order, + cheapest_order_choose, weighted_order_choose, + debug_dump_object, Wallet, BitcoinCoreWallet, + estimate_tx_fee) + +log = get_log() + + +def check_high_fee(total_fee_pc): + WARNING_THRESHOLD = 0.02 # 2% + if total_fee_pc > WARNING_THRESHOLD: + print('\n'.join(['=' * 60] * 3)) + print('WARNING ' * 6) + print('\n'.join(['=' * 60] * 1)) + print('OFFERED COINJOIN FEE IS UNUSUALLY HIGH. DOUBLE/TRIPLE CHECK.') + print('\n'.join(['=' * 60] * 1)) + print('WARNING ' * 6) + print('\n'.join(['=' * 60] * 3)) + + +def main(): + parser = OptionParser( + usage= + 'usage: %prog [options] [wallet file / fromaccount] [amount] [destaddr]', + description='Sends a single payment from a given mixing depth of your ' + + + 'wallet to an given address using coinjoin and then switches off. Also sends from bitcoinqt. ' + + + 'Setting amount to zero will do a sweep, where the entire mix depth is emptied') + parser.add_option( + '-f', + '--txfee', + action='store', + type='int', + dest='txfee', + default=-1, + help= + 'number of satoshis per participant to use as the initial estimate ' + + 'for the total transaction fee, default=dynamically estimated, note that this is adjusted ' + + + 'based on the estimated fee calculated after tx construction, based on ' + + 'policy set in joinmarket.cfg.') + parser.add_option( + '-w', + '--wait-time', + action='store', + type='float', + dest='waittime', + help='wait time in seconds to allow orders to arrive, default=15', + default=15) + parser.add_option( + '-N', + '--makercount', + action='store', + type='int', + dest='makercount', + help='how many makers to coinjoin with, default random from 4 to 6', + default=random.randint(4, 6)) + parser.add_option('-p', + '--port', + type='int', + dest='daemonport', + help='port on which joinmarketd is running', + default='12345') + parser.add_option( + '-C', + '--choose-cheapest', + action='store_true', + dest='choosecheapest', + default=False, + help= + 'override weightened offers picking and choose cheapest. this might reduce anonymity.') + parser.add_option( + '-P', + '--pick-orders', + action='store_true', + dest='pickorders', + default=False, + help= + 'manually pick which orders to take. doesn\'t work while sweeping.') + parser.add_option('-m', + '--mixdepth', + action='store', + type='int', + dest='mixdepth', + help='mixing depth to spend from, default=0', + default=0) + parser.add_option('-a', + '--amtmixdepths', + action='store', + type='int', + dest='amtmixdepths', + help='number of mixdepths in wallet, default 5', + default=5) + parser.add_option('-g', + '--gap-limit', + type="int", + action='store', + dest='gaplimit', + help='gap limit for wallet, default=6', + default=6) + parser.add_option('--yes', + action='store_true', + dest='answeryes', + default=False, + help='answer yes to everything') + parser.add_option( + '--rpcwallet', + action='store_true', + dest='userpcwallet', + default=False, + help=('Use the Bitcoin Core wallet through json rpc, instead ' + 'of the internal joinmarket wallet. Requires ' + 'blockchain_source=json-rpc')) + (options, args) = parser.parse_args() + + if len(args) < 3: + parser.error('Needs a wallet, amount and destination address') + sys.exit(0) + wallet_name = args[0] + amount = int(args[1]) + destaddr = args[2] + + load_program_config() + jm_single().maker_timeout_sec = 5 + addr_valid, errormsg = validate_address(destaddr) + if not addr_valid: + print('ERROR: Address invalid. ' + errormsg) + return + + chooseOrdersFunc = None + if options.pickorders: + chooseOrdersFunc = pick_order + if amount == 0: + print('WARNING: You may have to pick offers multiple times') + print('WARNING: due to manual offer picking while sweeping') + elif options.choosecheapest: + chooseOrdersFunc = cheapest_order_choose + else: # choose randomly (weighted) + chooseOrdersFunc = weighted_order_choose + + # Dynamically estimate a realistic fee if it currently is the default value. + # At this point we do not know even the number of our own inputs, so + # we guess conservatively with 2 inputs and 2 outputs each + if options.txfee == -1: + options.txfee = max(options.txfee, estimate_tx_fee(2, 2)) + log.debug("Estimated miner/tx fee for each cj participant: " + str( + options.txfee)) + assert (options.txfee >= 0) + + log.debug('starting sendpayment') + + if not options.userpcwallet: + wallet = Wallet(wallet_name, options.amtmixdepths, options.gaplimit) + else: + wallet = BitcoinCoreWallet(fromaccount=wallet_name) + jm_single().bc_interface.sync_wallet(wallet) + + taker = Taker(wallet, + options.mixdepth, + amount, + options.makercount, + order_chooser=chooseOrdersFunc, + external_addr=destaddr) + clientfactory = JMTakerClientProtocolFactory(taker) + start_reactor("localhost", options.daemonport, clientfactory) + + +if __name__ == "__main__": + main() + print('done')