diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..712da33 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +*.pyc +*.swp +.cache/ +.coverage +blockchain.cache +env +htmlcov/ +joinmarket.cfg +cmttools/commitments.json +blacklist + diff --git a/base/__init__.py b/base/__init__.py new file mode 100644 index 0000000..6bc56ed --- /dev/null +++ b/base/__init__.py @@ -0,0 +1,6 @@ +from __future__ import print_function + +from .support import (get_log, chunks, debug_silence, debug_dump_object, + joinmarket_alert, core_alert) +from commands import * + diff --git a/base/commands.py b/base/commands.py index 7dfe709..6ddc55c 100644 --- a/base/commands.py +++ b/base/commands.py @@ -1,3 +1,9 @@ +from __future__ import print_function +""" +Commands defining client-server (daemon) +messaging protocol (*not* Joinmarket p2p protocol). +Used for AMP asynchronous messages. +""" from twisted.protocols.amp import Integer, String, Unicode, Boolean, Command class DaemonNotReady(Exception): diff --git a/bitcoin/bci.py b/bitcoin/bci.py index 05b1829..c09a19b 100644 --- a/bitcoin/bci.py +++ b/bitcoin/bci.py @@ -4,7 +4,7 @@ import random import sys import time import platform -from joinmarketclient.support import get_log +from base.support import get_log if platform.system() == "Windows": import ssl import urllib2 diff --git a/bitcoin/secp256k1_main.py b/bitcoin/secp256k1_main.py index 6df66bc..6879b64 100644 --- a/bitcoin/secp256k1_main.py +++ b/bitcoin/secp256k1_main.py @@ -1,7 +1,5 @@ #!/usr/bin/python from __future__ import print_function -from .py2specials import * -from .py3specials import * import binascii import hashlib import re @@ -52,6 +50,89 @@ ffi.compile() import _noncefunc from _noncefunc import ffi +if sys.version_info.major == 2: + string_types = (str, unicode) + string_or_bytes_types = string_types + int_types = (int, float, long) + + # Base switching + code_strings = { + 2: '01', + 10: '0123456789', + 16: '0123456789abcdef', + 32: 'abcdefghijklmnopqrstuvwxyz234567', + 58: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz', + 256: ''.join([chr(x) for x in range(256)]) + } + + def bin_dbl_sha256(s): + bytes_to_hash = from_string_to_bytes(s) + return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest() + + def lpad(msg, symbol, length): + if len(msg) >= length: + return msg + return symbol * (length - len(msg)) + msg + + def get_code_string(base): + if base in code_strings: + return code_strings[base] + else: + raise ValueError("Invalid base!") + + def changebase(string, frm, to, minlen=0): + if frm == to: + return lpad(string, get_code_string(frm)[0], minlen) + return encode(decode(string, frm), to, minlen) + + def bin_to_b58check(inp, magicbyte=0): + inp_fmtd = chr(int(magicbyte)) + inp + leadingzbytes = len(re.match('^\x00*', inp_fmtd).group(0)) + checksum = bin_dbl_sha256(inp_fmtd)[:4] + return '1' * leadingzbytes + changebase(inp_fmtd + checksum, 256, 58) + + def bytes_to_hex_string(b): + return b.encode('hex') + + def safe_from_hex(s): + return s.decode('hex') + + def from_int_to_byte(a): + return chr(a) + + def from_byte_to_int(a): + return ord(a) + + def from_string_to_bytes(a): + return a + + def safe_hexlify(a): + return binascii.hexlify(a) + + def encode(val, base, minlen=0): + base, minlen = int(base), int(minlen) + code_string = get_code_string(base) + result = "" + while val > 0: + result = code_string[val % base] + result + val //= base + return code_string[0] * max(minlen - len(result), 0) + result + + def decode(string, base): + base = int(base) + code_string = get_code_string(base) + result = 0 + if base == 16: + string = string.lower() + while len(string) > 0: + result *= base + result += code_string.find(string[0]) + string = string[1:] + return result + +else: + raise NotImplementedError("Only Python2 currently supported by btc interface") + def tweak_mul(point, scalar): """Temporary hack because Windows binding had a bug in tweak_mul. Can be removed when Windows binding is updated. diff --git a/client/__init__.py b/client/__init__.py index 253e962..eaedc55 100644 --- a/client/__init__.py +++ b/client/__init__.py @@ -8,10 +8,9 @@ import logging #be implemented as an interface in btc.py from btc import * -from .support import get_log, calc_cj_fee, debug_dump_object, \ - choose_sweep_orders, choose_orders, \ - pick_order, cheapest_order_choose, weighted_order_choose, \ - rand_norm_array, rand_pow_array, rand_exp_array, joinmarket_alert, core_alert +from .support import (calc_cj_fee, choose_sweep_orders, choose_orders, + pick_order, cheapest_order_choose, weighted_order_choose, + rand_norm_array, rand_pow_array, rand_exp_array) from .jsonrpc import JsonRpcError, JsonRpcConnectionError, JsonRpc from .old_mnemonic import mn_decode, mn_encode from .slowaes import decryptData, encryptData diff --git a/client/blockchaininterface.py b/client/blockchaininterface.py index f5a14b6..a40a7d3 100644 --- a/client/blockchaininterface.py +++ b/client/blockchaininterface.py @@ -21,9 +21,9 @@ import btc # This can be removed once CliJsonRpc is gone. import subprocess -from joinmarketclient.jsonrpc import JsonRpcConnectionError, JsonRpcError -from joinmarketclient.configure import get_p2pk_vbyte, jm_single -from joinmarketclient.support import get_log, chunks +from client.jsonrpc import JsonRpcConnectionError, JsonRpcError +from client.configure import get_p2pk_vbyte, jm_single +from base.support import get_log, chunks log = get_log() @@ -682,7 +682,7 @@ class BitcoinCoreInterface(BlockchainInterface): sys.exit(0) def sync_addresses(self, wallet): - from joinmarketclient.wallet import BitcoinCoreWallet + from client.wallet import BitcoinCoreWallet if isinstance(wallet, BitcoinCoreWallet): return @@ -812,7 +812,7 @@ class BitcoinCoreInterface(BlockchainInterface): self.wallet_synced = True def sync_unspent(self, wallet): - from joinmarketclient.wallet import BitcoinCoreWallet + from client.wallet import BitcoinCoreWallet if isinstance(wallet, BitcoinCoreWallet): return diff --git a/client/btc.py b/client/btc.py index 76c400d..4aceb1b 100644 --- a/client/btc.py +++ b/client/btc.py @@ -6,7 +6,7 @@ BTC_P2PK_VBYTE = {"mainnet": 0x00, "testnet": 0x6f} BTC_P2SH_VBYTE = {"mainnet": 0x05, "testnet": 0xc4} PODLE_COMMIT_FILE = None -from .support import get_log +from base.support import get_log import binascii, sys, re, hashlib, base64 from pprint import pformat log = get_log() @@ -14,89 +14,6 @@ log = get_log() #Required only for PoDLE calculation: N = 115792089237316195423570985008687907852837564279074904382605163141518161494337 -if sys.version_info.major == 2: - string_types = (str, unicode) - string_or_bytes_types = string_types - int_types = (int, float, long) - - # Base switching - code_strings = { - 2: '01', - 10: '0123456789', - 16: '0123456789abcdef', - 32: 'abcdefghijklmnopqrstuvwxyz234567', - 58: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz', - 256: ''.join([chr(x) for x in range(256)]) - } - - def bin_dbl_sha256(s): - bytes_to_hash = from_string_to_bytes(s) - return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest() - - def lpad(msg, symbol, length): - if len(msg) >= length: - return msg - return symbol * (length - len(msg)) + msg - - def get_code_string(base): - if base in code_strings: - return code_strings[base] - else: - raise ValueError("Invalid base!") - - def changebase(string, frm, to, minlen=0): - if frm == to: - return lpad(string, get_code_string(frm)[0], minlen) - return encode(decode(string, frm), to, minlen) - - def bin_to_b58check(inp, magicbyte=0): - inp_fmtd = chr(int(magicbyte)) + inp - leadingzbytes = len(re.match('^\x00*', inp_fmtd).group(0)) - checksum = bin_dbl_sha256(inp_fmtd)[:4] - return '1' * leadingzbytes + changebase(inp_fmtd + checksum, 256, 58) - - def bytes_to_hex_string(b): - return b.encode('hex') - - def safe_from_hex(s): - return s.decode('hex') - - def from_int_to_byte(a): - return chr(a) - - def from_byte_to_int(a): - return ord(a) - - def from_string_to_bytes(a): - return a - - def safe_hexlify(a): - return binascii.hexlify(a) - - def encode(val, base, minlen=0): - base, minlen = int(base), int(minlen) - code_string = get_code_string(base) - result = "" - while val > 0: - result = code_string[val % base] + result - val //= base - return code_string[0] * max(minlen - len(result), 0) + result - - def decode(string, base): - base = int(base) - code_string = get_code_string(base) - result = 0 - if base == 16: - string = string.lower() - while len(string) > 0: - result *= base - result += code_string.find(string[0]) - string = string[1:] - return result - -else: - raise NotImplementedError("Only Python2 currently supported by btc interface") - interface = "joinmarket-joinmarket" try: diff --git a/client/client_protocol.py b/client/client_protocol.py index 88ebbcd..0cd4e3e 100644 --- a/client/client_protocol.py +++ b/client/client_protocol.py @@ -14,7 +14,7 @@ import string import time import hashlib import os -from joinmarketclient import (Taker, Wallet, jm_single, get_irc_mchannels, +from client import (Taker, Wallet, jm_single, get_irc_mchannels, load_program_config, get_log) import btc diff --git a/client/configure.py b/client/configure.py index 63392b3..e928fba 100644 --- a/client/configure.py +++ b/client/configure.py @@ -10,9 +10,9 @@ import sys from ConfigParser import SafeConfigParser, NoOptionError import btc -from joinmarketclient.jsonrpc import JsonRpc -from joinmarketclient.support import get_log, joinmarket_alert, core_alert, debug_silence -from joinmarketclient.podle import set_commitment_file +from client.jsonrpc import JsonRpc +from base.support import get_log, joinmarket_alert, core_alert, debug_silence +from client.podle import set_commitment_file log = get_log() @@ -341,9 +341,9 @@ def load_program_config(config_path=None, bs=None): def get_blockchain_interface_instance(_config): # todo: refactor joinmarket module to get rid of loops # importing here is necessary to avoid import loops - from joinmarketclient.blockchaininterface import BitcoinCoreInterface, \ + from client.blockchaininterface import BitcoinCoreInterface, \ RegtestBitcoinCoreInterface, BlockrInterface, ElectrumWalletInterface - from joinmarketclient.blockchaininterface import CliJsonRpc + from client.blockchaininterface import CliJsonRpc source = _config.get("BLOCKCHAIN", "blockchain_source") network = get_network() diff --git a/client/support.py b/client/support.py index 242bac0..b2cffb8 100644 --- a/client/support.py +++ b/client/support.py @@ -5,63 +5,15 @@ import sys import logging import pprint import random - +from base.support import get_log from decimal import Decimal from math import exp -# todo: this was the date format used in the original debug(). Use it? -# logging.basicConfig(filename='logs/joinmarket.log', -# stream=sys.stdout, -# level=logging.DEBUG, -# format='%(asctime)s %(message)s', -# dateformat='[%Y/%m/%d %H:%M:%S] ') - -logFormatter = logging.Formatter( - "%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s") -log = logging.getLogger('joinmarket') -log.setLevel(logging.DEBUG) - ORDER_KEYS = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', 'cjfee'] -joinmarket_alert = [''] -core_alert = [''] -debug_silence = [False] - - -#consoleHandler = logging.StreamHandler(stream=sys.stdout) -class JoinMarketStreamHandler(logging.StreamHandler): - - def __init__(self, stream): - super(JoinMarketStreamHandler, self).__init__(stream) - - def emit(self, record): - if joinmarket_alert[0]: - print('JoinMarket Alert Message: ' + joinmarket_alert[0]) - if core_alert[0]: - print('Core Alert Message: ' + core_alert[0]) - if not debug_silence[0]: - super(JoinMarketStreamHandler, self).emit(record) - - -consoleHandler = JoinMarketStreamHandler(stream=sys.stdout) -consoleHandler.setFormatter(logFormatter) -log.addHandler(consoleHandler) - -# log = logging.getLogger('joinmarket') -# log.addHandler(logging.NullHandler()) - -log.debug('hello joinmarket') - - -def get_log(): - """ - provides joinmarket logging instance - :return: log instance - """ - return log - +log = get_log() """ Random functions - replacing some NumPy features @@ -107,10 +59,6 @@ def rand_weighted_choice(n, p_arr): # End random functions - -def chunks(d, n): - return [d[x:x + n] for x in xrange(0, len(d), n)] - def select(unspent, value): """Default coin selection algorithm. """ @@ -409,21 +357,3 @@ def choose_sweep_orders(db, log.debug('cj amount = ' + str(cj_amount)) return result, cj_amount, total_fee - -def debug_dump_object(obj, skip_fields=None): - if skip_fields is None: - skip_fields = [] - log.debug('Class debug dump, name:' + obj.__class__.__name__) - for k, v in obj.__dict__.iteritems(): - if k in skip_fields: - continue - if k == 'password' or k == 'given_password': - continue - log.debug('key=' + k) - if isinstance(v, str): - log.debug('string: len:' + str(len(v))) - log.debug(v) - elif isinstance(v, dict) or isinstance(v, list): - log.debug(pprint.pformat(v)) - else: - log.debug(str(v)) diff --git a/client/taker.py b/client/taker.py index 0fb309f..ba5f9a4 100644 --- a/client/taker.py +++ b/client/taker.py @@ -9,11 +9,11 @@ import time import copy import btc -from joinmarketclient.configure import jm_single, get_p2pk_vbyte, donation_address -from joinmarketclient.support import (get_log, calc_cj_fee, weighted_order_choose, - choose_orders) -from joinmarketclient.wallet import estimate_tx_fee -from joinmarketclient.podle import (generate_podle, get_podle_commitments, +from client.configure import jm_single, get_p2pk_vbyte, donation_address +from base.support import get_log +from client.support import calc_cj_fee, weighted_order_choose, choose_orders +from client.wallet import estimate_tx_fee +from client.podle import (generate_podle, get_podle_commitments, PoDLE, PoDLEError) jlog = get_log() @@ -94,14 +94,14 @@ class Taker(object): try: self.my_cj_addr = self.wallet.get_external_addr(self.mixdepth + 1) except: - self.taker_error_callback("ABORT", "Failed to get an address") + self.taker_info_callback("ABORT", "Failed to get an address") return False self.my_change_addr = None if self.cjamount != 0: try: self.my_change_addr = self.wallet.get_internal_addr(self.mixdepth) except: - self.taker_error_callback("ABORT", "Failed to get a change address") + self.taker_info_callback("ABORT", "Failed to get a change address") return False #TODO sweep, doesn't apply here self.total_txfee = 2 * self.txfee_default * self.n_counterparties @@ -116,7 +116,7 @@ class Taker(object): self.input_utxos = self.wallet.select_utxos(self.mixdepth, total_amount) except Exception as e: - self.taker_error_callback("ABORT", + self.taker_info_callback("ABORT", "Unable to select sufficient coins: " + repr(e)) return False self.utxos = {None: self.input_utxos.keys()} diff --git a/client/wallet.py b/client/wallet.py index 1931db1..d887c10 100644 --- a/client/wallet.py +++ b/client/wallet.py @@ -9,12 +9,11 @@ from ConfigParser import NoSectionError from getpass import getpass import btc -from joinmarketclient.slowaes import decryptData -from joinmarketclient.blockchaininterface import BitcoinCoreInterface, RegtestBitcoinCoreInterface -from joinmarketclient.configure import jm_single, get_network, get_p2pk_vbyte - -from joinmarketclient.support import get_log, select_gradual, select_greedy, \ - select_greediest, select +from client.slowaes import decryptData +from client.blockchaininterface import BitcoinCoreInterface, RegtestBitcoinCoreInterface +from client.configure import jm_single, get_network, get_p2pk_vbyte +from base.support import get_log +from client.support import select_gradual, select_greedy,select_greediest, select log = get_log() diff --git a/daemon/__init__.py b/daemon/__init__.py index 6812c4b..edb90e7 100644 --- a/daemon/__init__.py +++ b/daemon/__init__.py @@ -5,10 +5,10 @@ from protocol import * from .enc_wrapper import as_init_encryption, decode_decrypt, \ encrypt_encode, init_keypair, init_pubkey, get_pubkey, NaclError from .irc import IRCMessageChannel, B_PER_SEC -from .support import get_log +from base.support import get_log from .message_channel import MessageChannel, MessageChannelCollection from .orderbookwatch import OrderbookWatch -import commands +from base import commands # Set default logging handler to avoid "No handler found" warnings. try: diff --git a/daemon/irc.py b/daemon/irc.py index cc36353..e192cda 100644 --- a/daemon/irc.py +++ b/daemon/irc.py @@ -9,10 +9,10 @@ import time import Queue -from joinmarketdaemon.message_channel import MessageChannel -from joinmarketdaemon.support import get_log, chunks -from joinmarketdaemon.socks import socksocket, setdefaultproxy, PROXY_TYPE_SOCKS5 -from joinmarketdaemon.protocol import * +from daemon.message_channel import MessageChannel +from base.support import get_log, chunks +from daemon.socks import socksocket, setdefaultproxy, PROXY_TYPE_SOCKS5 +from daemon.protocol import * MAX_PRIVMSG_LEN = 450 PING_INTERVAL = 300 PING_TIMEOUT = 60 diff --git a/daemon/message_channel.py b/daemon/message_channel.py index a570a9d..32c7ebc 100644 --- a/daemon/message_channel.py +++ b/daemon/message_channel.py @@ -1,12 +1,12 @@ #! /usr/bin/env python from __future__ import print_function import base64, abc, threading, time -from joinmarketdaemon import ( +from daemon import ( encrypt_encode, decode_decrypt, COMMAND_PREFIX, ORDER_KEYS, NICK_HASH_LENGTH, NICK_MAX_ENCODED, JM_VERSION, JOINMARKET_NICK_HEADER, nickname, plaintext_commands, encrypted_commands, commitment_broadcast_list, offername_list, public_commands, private_commands) -from joinmarketdaemon.support import get_log +from base.support import get_log from functools import wraps log = get_log() diff --git a/daemon/orderbookwatch.py b/daemon/orderbookwatch.py index edee599..ce5fe10 100644 --- a/daemon/orderbookwatch.py +++ b/daemon/orderbookwatch.py @@ -11,9 +11,9 @@ import threading import json from decimal import InvalidOperation, Decimal -from joinmarketdaemon.protocol import JM_VERSION -from joinmarketdaemon.support import get_log, joinmarket_alert, DUST_THRESHOLD -from joinmarketdaemon.irc import B_PER_SEC +from daemon.protocol import JM_VERSION +from base.support import get_log, joinmarket_alert, DUST_THRESHOLD +from daemon.irc import B_PER_SEC log = get_log() diff --git a/joinmarketd.py b/joinmarketd.py index c03632e..eb37395 100644 --- a/joinmarketd.py +++ b/joinmarketd.py @@ -1,13 +1,13 @@ #! /usr/bin/env python from __future__ import print_function import sys -from joinmarketdaemon import (IRCMessageChannel, MessageChannelCollection, +from daemon import (IRCMessageChannel, MessageChannelCollection, OrderbookWatch, as_init_encryption, init_pubkey, NaclError, init_keypair, COMMAND_PREFIX, ORDER_KEYS, NICK_HASH_LENGTH, NICK_MAX_ENCODED, JM_VERSION, JOINMARKET_NICK_HEADER) -from joinmarketdaemon.commands import * +from base.commands import * from twisted.protocols import amp from twisted.internet import reactor from twisted.internet.protocol import ServerFactory diff --git a/logs/.gitignore b/logs/.gitignore new file mode 100644 index 0000000..b9f50b1 --- /dev/null +++ b/logs/.gitignore @@ -0,0 +1,4 @@ +# Ignore all logs +*.log +# Except this file +!.gitignore \ No newline at end of file diff --git a/sendpayment.py b/sendpayment.py index a812043..9d27346 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -25,14 +25,16 @@ from optparse import OptionParser import time -from joinmarketclient import (Taker, load_program_config, +from client import (Taker, load_program_config, JMTakerClientProtocolFactory, start_reactor, - validate_address, jm_single, get_log, + validate_address, jm_single, choose_orders, choose_sweep_orders, pick_order, cheapest_order_choose, weighted_order_choose, - debug_dump_object, Wallet, BitcoinCoreWallet, + Wallet, BitcoinCoreWallet, estimate_tx_fee) +from base.support import get_log, debug_dump_object + log = get_log() diff --git a/test/regtest_joinmarket.cfg b/test/regtest_joinmarket.cfg new file mode 100644 index 0000000..50ba79d --- /dev/null +++ b/test/regtest_joinmarket.cfg @@ -0,0 +1,37 @@ +#NOTE: This configuration file is for testing with regtest only +#For mainnet usage, running a JoinMarket script will create the default file +[BLOCKCHAIN] +blockchain_source = regtest +rpc_host = localhost +rpc_port = 18332 +rpc_user = bitcoinrpc +rpc_password = 123456abcdef +network = testnet +bitcoin_cli_cmd = bitcoin-cli +notify_port = 62612 +[MESSAGING] +host = localhost, localhost +hostid = localhost1, localhost2 +channel = joinmarket-pit, joinmarket-pit +port = 6667, 6668 +usessl = false, false +socks5 = false, false +socks5_host = localhost, localhost +socks5_port = 9150, 9150 +[POLICY] +# for dust sweeping, try merge_algorithm = gradual +# for more rapid dust sweeping, try merge_algorithm = greedy +# for most rapid dust sweeping, try merge_algorithm = greediest +# but don't forget to bump your miner fees! +merge_algorithm = default +# the fee estimate is based on a projection of how many satoshis +# per kB are needed to get in one of the next N blocks, N set here +# as the value of 'tx_fees'. This estimate can be extremely high +# if you set N=1, so we choose N=3 for a more reasonable figure, +# as our default. Note that for clients not using a local blockchain +# instance, we retrieve an estimate from the API at cointape.com, currently. +tx_fees = 3 +taker_utxo_retries = 3 +taker_utxo_age = 1 +taker_utxo_amtpercent = 20 +accept_commitment_broadcasts = 1