refactored inputs, added dirs, sendpayment working

.gitignore

@@ -0,0 +1,11 @@
+*.pyc
+*.swp
+.cache/
+.coverage
+blockchain.cache
+env
+htmlcov/
+joinmarket.cfg
+cmttools/commitments.json
+blacklist
+

base/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import print_function
+
+from .support import (get_log, chunks, debug_silence, debug_dump_object,
+                      joinmarket_alert, core_alert)
+from commands import *
+

base/commands.py

@@ -1,3 +1,9 @@
+from __future__ import print_function
+"""
+Commands defining client-server (daemon)
+messaging protocol (*not* Joinmarket p2p protocol).
+Used for AMP asynchronous messages.
+"""
 from twisted.protocols.amp import Integer, String, Unicode, Boolean, Command
 
 class DaemonNotReady(Exception):

bitcoin/bci.py

@@ -4,7 +4,7 @@ import random
 import sys
 import time
 import platform
-from joinmarketclient.support import get_log
+from base.support import get_log
 if platform.system() == "Windows":
     import ssl
     import urllib2

bitcoin/secp256k1_main.py

@@ -1,7 +1,5 @@
 #!/usr/bin/python
 from __future__ import print_function
-from .py2specials import *
-from .py3specials import *
 import binascii
 import hashlib
 import re
@@ -52,6 +50,89 @@ ffi.compile()
 import _noncefunc
 from _noncefunc import ffi
 
+if sys.version_info.major == 2:
+    string_types = (str, unicode)
+    string_or_bytes_types = string_types
+    int_types = (int, float, long)
+
+    # Base switching
+    code_strings = {
+        2: '01',
+        10: '0123456789',
+        16: '0123456789abcdef',
+        32: 'abcdefghijklmnopqrstuvwxyz234567',
+        58: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz',
+        256: ''.join([chr(x) for x in range(256)])
+    }
+
+    def bin_dbl_sha256(s):
+        bytes_to_hash = from_string_to_bytes(s)
+        return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest()
+
+    def lpad(msg, symbol, length):
+        if len(msg) >= length:
+            return msg
+        return symbol * (length - len(msg)) + msg
+
+    def get_code_string(base):
+        if base in code_strings:
+            return code_strings[base]
+        else:
+            raise ValueError("Invalid base!")
+
+    def changebase(string, frm, to, minlen=0):
+        if frm == to:
+            return lpad(string, get_code_string(frm)[0], minlen)
+        return encode(decode(string, frm), to, minlen)
+
+    def bin_to_b58check(inp, magicbyte=0):
+        inp_fmtd = chr(int(magicbyte)) + inp
+        leadingzbytes = len(re.match('^\x00*', inp_fmtd).group(0))
+        checksum = bin_dbl_sha256(inp_fmtd)[:4]
+        return '1' * leadingzbytes + changebase(inp_fmtd + checksum, 256, 58)
+
+    def bytes_to_hex_string(b):
+        return b.encode('hex')
+
+    def safe_from_hex(s):
+        return s.decode('hex')
+
+    def from_int_to_byte(a):
+        return chr(a)
+
+    def from_byte_to_int(a):
+        return ord(a)
+
+    def from_string_to_bytes(a):
+        return a
+
+    def safe_hexlify(a):
+        return binascii.hexlify(a)
+
+    def encode(val, base, minlen=0):
+        base, minlen = int(base), int(minlen)
+        code_string = get_code_string(base)
+        result = ""
+        while val > 0:
+            result = code_string[val % base] + result
+            val //= base
+        return code_string[0] * max(minlen - len(result), 0) + result
+
+    def decode(string, base):
+        base = int(base)
+        code_string = get_code_string(base)
+        result = 0
+        if base == 16:
+            string = string.lower()
+        while len(string) > 0:
+            result *= base
+            result += code_string.find(string[0])
+            string = string[1:]
+        return result
+
+else:
+    raise NotImplementedError("Only Python2 currently supported by btc interface")
+
 def tweak_mul(point, scalar):
     """Temporary hack because Windows binding had a bug in tweak_mul.
     Can be removed when Windows binding is updated.

client/__init__.py

@@ -8,10 +8,9 @@ import logging
 #be implemented as an interface in btc.py
 from btc import *
 
-from .support import get_log, calc_cj_fee, debug_dump_object, \
+from .support import (calc_cj_fee, choose_sweep_orders, choose_orders,
-    choose_sweep_orders, choose_orders, \
+                      pick_order, cheapest_order_choose, weighted_order_choose,
-    pick_order, cheapest_order_choose, weighted_order_choose, \
+                      rand_norm_array, rand_pow_array, rand_exp_array)
-    rand_norm_array, rand_pow_array, rand_exp_array, joinmarket_alert, core_alert
 from .jsonrpc import JsonRpcError, JsonRpcConnectionError, JsonRpc
 from .old_mnemonic import mn_decode, mn_encode
 from .slowaes import decryptData, encryptData

client/blockchaininterface.py

@@ -21,9 +21,9 @@ import btc
 # This can be removed once CliJsonRpc is gone.
 import subprocess
 
-from joinmarketclient.jsonrpc import JsonRpcConnectionError, JsonRpcError
+from client.jsonrpc import JsonRpcConnectionError, JsonRpcError
-from joinmarketclient.configure import get_p2pk_vbyte, jm_single
+from client.configure import get_p2pk_vbyte, jm_single
-from joinmarketclient.support import get_log, chunks
+from base.support import get_log, chunks
 
 log = get_log()
 
@@ -682,7 +682,7 @@ class BitcoinCoreInterface(BlockchainInterface):
             sys.exit(0)
 
     def sync_addresses(self, wallet):
-        from joinmarketclient.wallet import BitcoinCoreWallet
+        from client.wallet import BitcoinCoreWallet
 
         if isinstance(wallet, BitcoinCoreWallet):
             return
@@ -812,7 +812,7 @@ class BitcoinCoreInterface(BlockchainInterface):
         self.wallet_synced = True
 
     def sync_unspent(self, wallet):
-        from joinmarketclient.wallet import BitcoinCoreWallet
+        from client.wallet import BitcoinCoreWallet
 
         if isinstance(wallet, BitcoinCoreWallet):
             return

client/btc.py

@@ -6,7 +6,7 @@ BTC_P2PK_VBYTE = {"mainnet": 0x00, "testnet": 0x6f}
 BTC_P2SH_VBYTE = {"mainnet": 0x05, "testnet": 0xc4}
 PODLE_COMMIT_FILE = None
 
-from .support import get_log
+from base.support import get_log
 import binascii, sys, re, hashlib, base64
 from pprint import pformat
 log = get_log()
@@ -14,89 +14,6 @@ log = get_log()
 #Required only for PoDLE calculation:
 N = 115792089237316195423570985008687907852837564279074904382605163141518161494337
 
-if sys.version_info.major == 2:
-    string_types = (str, unicode)
-    string_or_bytes_types = string_types
-    int_types = (int, float, long)
-
-    # Base switching
-    code_strings = {
-        2: '01',
-        10: '0123456789',
-        16: '0123456789abcdef',
-        32: 'abcdefghijklmnopqrstuvwxyz234567',
-        58: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz',
-        256: ''.join([chr(x) for x in range(256)])
-    }
-
-    def bin_dbl_sha256(s):
-        bytes_to_hash = from_string_to_bytes(s)
-        return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest()
-
-    def lpad(msg, symbol, length):
-        if len(msg) >= length:
-            return msg
-        return symbol * (length - len(msg)) + msg
-
-    def get_code_string(base):
-        if base in code_strings:
-            return code_strings[base]
-        else:
-            raise ValueError("Invalid base!")
-
-    def changebase(string, frm, to, minlen=0):
-        if frm == to:
-            return lpad(string, get_code_string(frm)[0], minlen)
-        return encode(decode(string, frm), to, minlen)
-
-    def bin_to_b58check(inp, magicbyte=0):
-        inp_fmtd = chr(int(magicbyte)) + inp
-        leadingzbytes = len(re.match('^\x00*', inp_fmtd).group(0))
-        checksum = bin_dbl_sha256(inp_fmtd)[:4]
-        return '1' * leadingzbytes + changebase(inp_fmtd + checksum, 256, 58)
-
-    def bytes_to_hex_string(b):
-        return b.encode('hex')
-
-    def safe_from_hex(s):
-        return s.decode('hex')
-
-    def from_int_to_byte(a):
-        return chr(a)
-
-    def from_byte_to_int(a):
-        return ord(a)
-
-    def from_string_to_bytes(a):
-        return a
-
-    def safe_hexlify(a):
-        return binascii.hexlify(a)
-
-    def encode(val, base, minlen=0):
-        base, minlen = int(base), int(minlen)
-        code_string = get_code_string(base)
-        result = ""
-        while val > 0:
-            result = code_string[val % base] + result
-            val //= base
-        return code_string[0] * max(minlen - len(result), 0) + result
-
-    def decode(string, base):
-        base = int(base)
-        code_string = get_code_string(base)
-        result = 0
-        if base == 16:
-            string = string.lower()
-        while len(string) > 0:
-            result *= base
-            result += code_string.find(string[0])
-            string = string[1:]
-        return result
-
-else:
-    raise NotImplementedError("Only Python2 currently supported by btc interface")
-
 interface = "joinmarket-joinmarket"
 
 try:

client/client_protocol.py

@@ -14,7 +14,7 @@ import string
 import time
 import hashlib
 import os
-from joinmarketclient import (Taker, Wallet, jm_single, get_irc_mchannels,
+from client import (Taker, Wallet, jm_single, get_irc_mchannels,
                         load_program_config, get_log)
 
 import btc

client/configure.py

@@ -10,9 +10,9 @@ import sys
 from ConfigParser import SafeConfigParser, NoOptionError
 
 import btc
-from joinmarketclient.jsonrpc import JsonRpc
+from client.jsonrpc import JsonRpc
-from joinmarketclient.support import get_log, joinmarket_alert, core_alert, debug_silence
+from base.support import get_log, joinmarket_alert, core_alert, debug_silence
-from joinmarketclient.podle import set_commitment_file
+from client.podle import set_commitment_file
 
 log = get_log()
 
@@ -341,9 +341,9 @@ def load_program_config(config_path=None, bs=None):
 def get_blockchain_interface_instance(_config):
     # todo: refactor joinmarket module to get rid of loops
     # importing here is necessary to avoid import loops
-    from joinmarketclient.blockchaininterface import BitcoinCoreInterface, \
+    from client.blockchaininterface import BitcoinCoreInterface, \
         RegtestBitcoinCoreInterface, BlockrInterface, ElectrumWalletInterface
-    from joinmarketclient.blockchaininterface import CliJsonRpc
+    from client.blockchaininterface import CliJsonRpc
 
     source = _config.get("BLOCKCHAIN", "blockchain_source")
     network = get_network()

client/support.py

@@ -5,63 +5,15 @@ import sys
 import logging
 import pprint
 import random
-
+from base.support import get_log
 from decimal import Decimal
 
 from math import exp
 
-# todo: this was the date format used in the original debug().  Use it?
-# logging.basicConfig(filename='logs/joinmarket.log',
-#                     stream=sys.stdout,
-#                     level=logging.DEBUG,
-#                     format='%(asctime)s %(message)s',
-#                     dateformat='[%Y/%m/%d %H:%M:%S] ')
-
-logFormatter = logging.Formatter(
-    "%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s]  %(message)s")
-log = logging.getLogger('joinmarket')
-log.setLevel(logging.DEBUG)
-
 ORDER_KEYS = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee',
               'cjfee']
 
-joinmarket_alert = ['']
+log = get_log()
-core_alert = ['']
-debug_silence = [False]
-
-
-#consoleHandler = logging.StreamHandler(stream=sys.stdout)
-class JoinMarketStreamHandler(logging.StreamHandler):
-
-    def __init__(self, stream):
-        super(JoinMarketStreamHandler, self).__init__(stream)
-
-    def emit(self, record):
-        if joinmarket_alert[0]:
-            print('JoinMarket Alert Message: ' + joinmarket_alert[0])
-        if core_alert[0]:
-            print('Core Alert Message: ' + core_alert[0])
-        if not debug_silence[0]:
-            super(JoinMarketStreamHandler, self).emit(record)
-
-
-consoleHandler = JoinMarketStreamHandler(stream=sys.stdout)
-consoleHandler.setFormatter(logFormatter)
-log.addHandler(consoleHandler)
-
-# log = logging.getLogger('joinmarket')
-# log.addHandler(logging.NullHandler())
-
-log.debug('hello joinmarket')
-
-
-def get_log():
-    """
-    provides joinmarket logging instance
-    :return: log instance
-    """
-    return log
-
 
 """
 Random functions - replacing some NumPy features
@@ -107,10 +59,6 @@ def rand_weighted_choice(n, p_arr):
 
 # End random functions
 
-
-def chunks(d, n):
-    return [d[x:x + n] for x in xrange(0, len(d), n)]
-
 def select(unspent, value):
     """Default coin selection algorithm.
     """
@@ -409,21 +357,3 @@ def choose_sweep_orders(db,
     log.debug('cj amount = ' + str(cj_amount))
     return result, cj_amount, total_fee
 
-
-def debug_dump_object(obj, skip_fields=None):
-    if skip_fields is None:
-        skip_fields = []
-    log.debug('Class debug dump, name:' + obj.__class__.__name__)
-    for k, v in obj.__dict__.iteritems():
-        if k in skip_fields:
-            continue
-        if k == 'password' or k == 'given_password':
-            continue
-        log.debug('key=' + k)
-        if isinstance(v, str):
-            log.debug('string: len:' + str(len(v)))
-            log.debug(v)
-        elif isinstance(v, dict) or isinstance(v, list):
-            log.debug(pprint.pformat(v))
-        else:
-            log.debug(str(v))

client/taker.py

@@ -9,11 +9,11 @@ import time
 import copy
 
 import btc
-from joinmarketclient.configure import jm_single, get_p2pk_vbyte, donation_address
+from client.configure import jm_single, get_p2pk_vbyte, donation_address
-from joinmarketclient.support import (get_log, calc_cj_fee, weighted_order_choose,
+from base.support import get_log
-                                choose_orders)
+from client.support import calc_cj_fee, weighted_order_choose, choose_orders
-from joinmarketclient.wallet import estimate_tx_fee
+from client.wallet import estimate_tx_fee
-from joinmarketclient.podle import (generate_podle, get_podle_commitments,
+from client.podle import (generate_podle, get_podle_commitments,
                                     PoDLE, PoDLEError)
 jlog = get_log()
 
@@ -94,14 +94,14 @@ class Taker(object):
             try:
                 self.my_cj_addr = self.wallet.get_external_addr(self.mixdepth + 1)
             except:
-                self.taker_error_callback("ABORT", "Failed to get an address")
+                self.taker_info_callback("ABORT", "Failed to get an address")
                 return False
         self.my_change_addr = None
         if self.cjamount != 0:
             try:
                 self.my_change_addr = self.wallet.get_internal_addr(self.mixdepth)
             except:
-                self.taker_error_callback("ABORT", "Failed to get a change address")
+                self.taker_info_callback("ABORT", "Failed to get a change address")
                 return False
         #TODO sweep, doesn't apply here
         self.total_txfee = 2 * self.txfee_default * self.n_counterparties
@@ -116,7 +116,7 @@ class Taker(object):
             self.input_utxos = self.wallet.select_utxos(self.mixdepth,
                                                         total_amount)
         except Exception as e:
-            self.taker_error_callback("ABORT",
+            self.taker_info_callback("ABORT",
                                 "Unable to select sufficient coins: " + repr(e))
             return False
         self.utxos = {None: self.input_utxos.keys()}

client/wallet.py

@@ -9,12 +9,11 @@ from ConfigParser import NoSectionError
 from getpass import getpass
 
 import btc
-from joinmarketclient.slowaes import decryptData
+from client.slowaes import decryptData
-from joinmarketclient.blockchaininterface import BitcoinCoreInterface, RegtestBitcoinCoreInterface
+from client.blockchaininterface import BitcoinCoreInterface, RegtestBitcoinCoreInterface
-from joinmarketclient.configure import jm_single, get_network, get_p2pk_vbyte
+from client.configure import jm_single, get_network, get_p2pk_vbyte
-
+from base.support import get_log
-from joinmarketclient.support import get_log, select_gradual, select_greedy, \
+from client.support import select_gradual, select_greedy,select_greediest, select
-    select_greediest, select
 
 log = get_log()
 

daemon/__init__.py

@@ -5,10 +5,10 @@ from protocol import *
 from .enc_wrapper import as_init_encryption, decode_decrypt, \
     encrypt_encode, init_keypair, init_pubkey, get_pubkey, NaclError
 from .irc import IRCMessageChannel, B_PER_SEC
-from .support import get_log
+from base.support import get_log
 from .message_channel import MessageChannel, MessageChannelCollection
 from .orderbookwatch import OrderbookWatch
-import commands
+from base import commands
 
 # Set default logging handler to avoid "No handler found" warnings.
 try:

daemon/irc.py

@@ -9,10 +9,10 @@ import time
 import Queue
 
 
-from joinmarketdaemon.message_channel import MessageChannel
+from daemon.message_channel import MessageChannel
-from joinmarketdaemon.support import get_log, chunks
+from base.support import get_log, chunks
-from joinmarketdaemon.socks import socksocket, setdefaultproxy, PROXY_TYPE_SOCKS5
+from daemon.socks import socksocket, setdefaultproxy, PROXY_TYPE_SOCKS5
-from joinmarketdaemon.protocol import *
+from daemon.protocol import *
 MAX_PRIVMSG_LEN = 450
 PING_INTERVAL = 300
 PING_TIMEOUT = 60

daemon/message_channel.py

@@ -1,12 +1,12 @@
 #! /usr/bin/env python
 from __future__ import print_function
 import base64, abc, threading, time
-from joinmarketdaemon import (
+from daemon import (
     encrypt_encode, decode_decrypt, COMMAND_PREFIX, ORDER_KEYS,
     NICK_HASH_LENGTH, NICK_MAX_ENCODED, JM_VERSION, JOINMARKET_NICK_HEADER,
     nickname, plaintext_commands, encrypted_commands, commitment_broadcast_list,
     offername_list, public_commands, private_commands)
-from joinmarketdaemon.support import get_log
+from base.support import get_log
 from functools import wraps
 
 log = get_log()

daemon/orderbookwatch.py

@@ -11,9 +11,9 @@ import threading
 import json
 from decimal import InvalidOperation, Decimal
 
-from joinmarketdaemon.protocol import JM_VERSION
+from daemon.protocol import JM_VERSION
-from joinmarketdaemon.support import get_log, joinmarket_alert, DUST_THRESHOLD
+from base.support import get_log, joinmarket_alert, DUST_THRESHOLD
-from joinmarketdaemon.irc import B_PER_SEC
+from daemon.irc import B_PER_SEC
 log = get_log()
 
 

joinmarketd.py

@@ -1,13 +1,13 @@
 #! /usr/bin/env python
 from __future__ import print_function
 import sys
-from joinmarketdaemon import (IRCMessageChannel, MessageChannelCollection,
+from daemon import (IRCMessageChannel, MessageChannelCollection,
                         OrderbookWatch, as_init_encryption, init_pubkey,
                         NaclError, init_keypair, COMMAND_PREFIX, ORDER_KEYS,
                         NICK_HASH_LENGTH, NICK_MAX_ENCODED, JM_VERSION,
                         JOINMARKET_NICK_HEADER)
 
-from joinmarketdaemon.commands import *
+from base.commands import *
 from twisted.protocols import amp
 from twisted.internet import reactor
 from twisted.internet.protocol import ServerFactory

logs/.gitignore

@@ -0,0 +1,4 @@
+# Ignore all logs
+*.log
+# Except this file
+!.gitignore

sendpayment.py

@@ -25,14 +25,16 @@ from optparse import OptionParser
 
 import time
 
-from joinmarketclient import (Taker, load_program_config,
+from client import (Taker, load_program_config,
                               JMTakerClientProtocolFactory, start_reactor,
-                              validate_address, jm_single, get_log,
+                              validate_address, jm_single,
                               choose_orders, choose_sweep_orders, pick_order,
                               cheapest_order_choose, weighted_order_choose,
-                              debug_dump_object, Wallet, BitcoinCoreWallet,
+                              Wallet, BitcoinCoreWallet,
                               estimate_tx_fee)
 
+from base.support import get_log, debug_dump_object
+
 log = get_log()
 
 

test/regtest_joinmarket.cfg

@@ -0,0 +1,37 @@
+#NOTE: This configuration file is for testing with regtest only
+#For mainnet usage, running a JoinMarket script will create the default file
+[BLOCKCHAIN]
+blockchain_source = regtest
+rpc_host = localhost
+rpc_port = 18332
+rpc_user = bitcoinrpc
+rpc_password = 123456abcdef
+network = testnet
+bitcoin_cli_cmd = bitcoin-cli
+notify_port = 62612
+[MESSAGING]
+host = localhost, localhost
+hostid = localhost1, localhost2
+channel = joinmarket-pit, joinmarket-pit
+port = 6667, 6668
+usessl = false, false
+socks5 = false, false
+socks5_host = localhost, localhost
+socks5_port = 9150, 9150
+[POLICY]
+# for dust sweeping, try merge_algorithm = gradual
+# for more rapid dust sweeping, try merge_algorithm = greedy
+# for most rapid dust sweeping, try merge_algorithm = greediest
+# but don't forget to bump your miner fees!
+merge_algorithm = default
+# the fee estimate is based on a projection of how many satoshis
+# per kB are needed to get in one of the next N blocks, N set here
+# as the value of 'tx_fees'. This estimate can be extremely high
+# if you set N=1, so we choose N=3 for a more reasonable figure,
+# as our default. Note that for clients not using a local blockchain
+# instance, we retrieve an estimate from the API at cointape.com, currently.
+tx_fees = 3
+taker_utxo_retries = 3
+taker_utxo_age = 1
+taker_utxo_amtpercent = 20
+accept_commitment_broadcasts = 1