From 80e17df4a6bf6929bb132b8f3c8017e42a4ee28d Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Wed, 5 Aug 2020 12:47:48 +0100 Subject: [PATCH] Add jmwalletd script as RPC server. Uses Klein to provide HTTP server support. Adds cookie based auth to requests (made JWT token based in later commits). Basic routes are: /unlock, /lock, /display, /create of wallet. Encapsulates WalletDaemon as a Service Add snicker receiver service start, stop Adds yg/maker function as stoppable service. Adds a JMShutdown command to the AMP protocol, allowing a clean shutdown of a long running bot (e.g. maker) by shutting down its message channel connections, without shutting down the entire process. Adds payment(direct send) request, first draft --- docs/JSON-RPC-API-using-jmwalletd.md | 118 ++++++++ jmbase/jmbase/commands.py | 7 + jmclient/jmclient/__init__.py | 9 +- jmclient/jmclient/client_protocol.py | 49 ++- jmclient/jmclient/snicker_receiver.py | 47 ++- jmclient/jmclient/wallet_utils.py | 33 +- jmclient/jmclient/yieldgenerator.py | 34 +++ jmdaemon/jmdaemon/daemon_protocol.py | 6 + jmdaemon/jmdaemon/irc.py | 12 +- scripts/jmwalletd.py | 420 ++++++++++++++++++++++++++ 10 files changed, 712 insertions(+), 23 deletions(-) create mode 100644 docs/JSON-RPC-API-using-jmwalletd.md create mode 100644 scripts/jmwalletd.py diff --git a/docs/JSON-RPC-API-using-jmwalletd.md b/docs/JSON-RPC-API-using-jmwalletd.md new file mode 100644 index 0000000..6601d3a --- /dev/null +++ b/docs/JSON-RPC-API-using-jmwalletd.md @@ -0,0 +1,118 @@ +## JSON-RPC API for Joinmarket using jmwalletd.py + +### Introduction - how to start the server + +After installing Joinmarket as per the [INSTALL GUIDE](INSTALL.md), navigate to the `scripts/` directory as usual and start the server with: + +``` +(jmvenv) $python jmwalletd.py +``` + +which with defaults will start serving the RPC over HTTP on port 28183. + +This HTTP server does *NOT* currently support multiple sessions; it is intended as a manager/daemon for all the Joinmarket services for a single user. + +#### Rules about making requests + +Currently authentication is done by providing a cookie on first request, which must then be reused to keep the same session. The cookie is sent in an HTTP header with name `b'JMCookie'`. This is fine for an early testing stage, but will be improved/reworked, and that will be documented here. + +GET requests are used in case no content or parameters need to be provided with the request. + +POST requests are used in case content or parameters are to be provided with the request, and they are provided as utf-8 encoded serialized JSON, in the *body* of the POST request. + +Note that for some methods, it's particularly important to deal with the HTTP response asynchronously, since it can take some time for wallet synchronization, service startup etc. to occur. + +### Methods + +#### `createwallet` + +Make a new wallet. The variable "wallettype" should be "sw" for native segwit wallets (now the Joinmarket default), otherwise a segwit legacy wallet (BIP49) will be created. + +* HTTP Request type: POST +* Route: `/wallet/create` +* POST body contents: {"walletname": walletname, "password": password, "wallettype": wallettype} +* Returns: on success, {"walletname": walletname, "already_loaded": False} + +(TODO some confusion over two different walletnames here, I need to check, but the wallet name sent by the caller will be used for the file name, I believe). + +#### `unlockwallet` + +Open an existing wallet using a password. + +* HTTP Request type: POST +* Route: `/wallet//unlock` +* POST body contents: {"password": password} +* Returns: on success, {"walletname": walletname, "already_loaded": True} + +(see previous on walletname, same applies here). + +#### `lockwallet` + +Stops the wallet service for the current wallet; meaning it cannot then be accessed without re-authentication. + +* HTTP Request type: GET +* Route: `/wallet//lock` +* Returns: on success, {"walletname": walletname} + +(see previous on walletname, same applies here). + +#### `displaywallet` + +Get JSON representation of wallet contents for wallet named `walletname`: + +* HTTP Request type: GET +* Route: `/wallet//display` +* Returns: a JSON object which is the entire wallet contents, mixdepth by mixdepth. + - Example output from a signet wallet is given at the bottom of the document. + +#### `maker/start` + +Starts the yield generator/maker service for the given wallet, using the IRC and tor network connections +in the backend (inside the process started with jmwalletd). +See Joinmarket yield generator config defaults in `jmclient.configure` module for info on the data that must +be specified in the POST body contents. + +* HTTP Request type: POST +* Route: `/wallet//maker/start` +* POST body contents: {"txfee", "cjfee_a", "cjfee_r", "ordertype", "minsize"] +* Returns: on success, {"walletname": walletname} + +(see previous on walletname, same applies here). + +#### `maker/stop` + +Stops the yieldgenerator/maker service if currently running for the given wallet. + +* HTTP Request type: GET +* Route: `/wallet//maker/start` +* Returns: on success, {"walletname": walletname} + +(see previous on walletname, same applies here). + +#### `snicker/start` + +Starts the SNICKER service (see [here](SNICKER.md)) for the given wallet. Note that this requires +no configuration for now, though that is likely to change. Also note this is not yet supported for +mainnet. + +* HTTP Request type: GET +* Route: `/wallet//snicker/start` +* Returns: on success, {"walletname": walletname} + +(see previous on walletname, same applies here). + +#### `snicker/stop` + +Stops the snicker service if currently running for the given wallet. + +* HTTP Request type: GET +* Route: `/wallet//snicker/start` +* Returns: on success, {"walletname": walletname} + +(see previous on walletname, same applies here). + +##### Example wallet display JSON output from signet wallet + +``` +{'wallet_name': 'JM wallet', 'total_balance': '0.15842426', 'accounts': [{'account': '0', 'account_balance': '0.00861458', 'branches': [{'branch': "external addresses\tm/84'/1'/0'/0\ttpubDFGxEsV7NvVc4h2XL4QEppZt3CrDiCFksP97H6YbFPmCTKM6KMP2xUxW57gAu7bzDfB3YTqnMeKQaQRS5GJM3xMcrhbi5AGsQUd7p4PLMDV", 'balance': '0.00000000', 'entries': [{'hd_path': "m/84'/1'/0'/0/4", 'address': 'tb1qzugshsm85x6luegyjc6mk5zces2zqr0j8m4zkd', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/0'/0/5", 'address': 'tb1qcwmdkg229ghmd8r3xgq4a9zxp459crws66n4ve', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/0'/0/6", 'address': 'tb1q7lv6dwex3mhwp32vhku0fvpar9faar2lu595su', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/0'/0/7", 'address': 'tb1qm42ltytvp22kj9efp995yu0r0r7x570d8j8crc', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/0'/0/8", 'address': 'tb1qwvux8g0khuvvkla3zaqdslj6xpgwtq7jlvwmgu', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/0'/0/9", 'address': 'tb1q3xr7l9nylsdlyqf9rkw0rg3f0yx6slguhtwpzp', 'amount': '0.00000000', 'labels': 'new'}]}, {'branch': "internal addresses\tm/84'/1'/0'/1\t", 'balance': '0.00861458', 'entries': [{'hd_path': "m/84'/1'/0'/1/7", 'address': 'tb1qjrzxkulgc5dnlyz0rjqj68zxgqjesqn839ue2w', 'amount': '0.00396839', 'labels': 'cj-out'}, {'hd_path': "m/84'/1'/0'/1/12", 'address': 'tb1qeqkk4te2t6gqt7jfgu8a9k4je2wwfw3d2m7gku', 'amount': '0.00464619', 'labels': 'non-cj-change'}]}]}, {'account': '1', 'account_balance': '0.09380968', 'branches': [{'branch': "external addresses\tm/84'/1'/1'/0\ttpubDE1TKa8tm3WWh4f9fV325BgYWX9i7WFMaQRd1C3tSFYU9RJEyE8w2Cw2KnhgXSKyjS4keeWAkc3iLEqp3pxUEG9T49RCtQiMpjuZM71FLpL", 'balance': '0.00000000', 'entries': [{'hd_path': "m/84'/1'/1'/0/0", 'address': 'tb1qd6qqg3uzk9sw88yhvpqpwt3tx5ls4hau3mwh3g', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/1'/0/1", 'address': 'tb1qhkrmqn9e4ldzlwna8w5w9l5vaw978zlrl54hmh', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/1'/0/2", 'address': 'tb1qp83afad8dl98w366vnvct0zc49qu33c2nfx386', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/1'/0/3", 'address': 'tb1qjv0elh4kn5yaywajedgcrf93ujzz3m3q7ld7k3', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/1'/0/4", 'address': 'tb1qk25u4ch7w0xylzh0krn4hefphe6xpyh0vc33sl', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/1'/0/5", 'address': 'tb1qs3ep9nlypwn43swv75zwv6lgl3wgsmha20g87p', 'amount': '0.00000000', 'labels': 'new'}]}, {'branch': "internal addresses\tm/84'/1'/1'/1\t", 'balance': '0.09380968', 'entries': [{'hd_path': "m/84'/1'/1'/1/44", 'address': 'tb1qgmgpk22ueq9xk8f722aqjnuwd6s3jv58nwwan2', 'amount': '0.00009631', 'labels': 'non-cj-change'}, {'hd_path': "m/84'/1'/1'/1/49", 'address': 'tb1qjq86y8nzvafv5dsde93zf0emv7yrsphvupv69e', 'amount': '0.00013383', 'labels': 'non-cj-change'}, {'hd_path': "m/84'/1'/1'/1/54", 'address': 'tb1q7lvxk407xs38t24hfzy7vprp9t7tfsemv4rfym', 'amount': '0.00371951', 'labels': 'non-cj-change'}, {'hd_path': "m/84'/1'/1'/1/56", 'address': 'tb1qn2azshrkcg0d7py5apgfr0jh29nt9w2fmx9fyy', 'amount': '0.08986003', 'labels': 'non-cj-change'}]}, {'branch': 'Imported keys\tm/0\t', 'balance': '0.00000000', 'entries': [{'hd_path': 'imported/1/0', 'address': 'tb1q8znprh8c85za3mpwzn3qf9m0vwqzjkfu4qdncy', 'amount': '0.00000000', 'labels': 'empty'}, {'hd_path': 'imported/1/1', 'address': 'tb1qu4ajg3enea90xxtjuwcurj3d6lkqrud8p7w0yu', 'amount': '0.00000000', 'labels': 'empty'}, {'hd_path': 'imported/1/2', 'address': 'tb1qg7saqx69yalcqshfr8mjndy0gpx2umxrwqs823', 'amount': '0.00000000', 'labels': 'empty'}]}]}, {'account': '2', 'account_balance': '0.05600000', 'branches': [{'branch': "external addresses\tm/84'/1'/2'/0\ttpubDF8K7wXCrRXX1CQLVZGwMvEg9YEWF2VRpM1tjCwpMZDRRqKjpJ5YaeaDaLkqN1D7YM4pkX32FcCnosbhLQz2BgRiPNNdybWuvSBKp72mJsJ", 'balance': '0.00000000', 'entries': [{'hd_path': "m/84'/1'/2'/0/0", 'address': 'tb1qw95x9m84t6hqcun560vqfk3yc6ptl4g9arsty0', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/2'/0/1", 'address': 'tb1qek4humez7rcwl53ly6uzr4mfwd0s2lu92e356q', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/2'/0/2", 'address': 'tb1qxne4hyyeq2vrh0dfzs56th29qsymp9eq5pljdc', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/2'/0/3", 'address': 'tb1qz3jk544j5vtwztznxfdwfgt8zcw77mjcut8vdz', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/2'/0/4", 'address': 'tb1qg902humlsuc5s6aua6ew3d893hlgcxr05ntpyd', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/2'/0/5", 'address': 'tb1qukz3l34ydy9snq8rkjaknk0ns04kfnlh34neqd', 'amount': '0.00000000', 'labels': 'new'}]}, {'branch': "internal addresses\tm/84'/1'/2'/1\t", 'balance': '0.05600000', 'entries': [{'hd_path': "m/84'/1'/2'/1/1", 'address': 'tb1qrtz5cwpneheg2v2v32wzc3h9yv0rzplxjtx9vc', 'amount': '0.00800000', 'labels': 'non-cj-change'}, {'hd_path': "m/84'/1'/2'/1/2", 'address': 'tb1qp4276g23y2w8g3367de25ustxkygjydmwk4fw2', 'amount': '0.00800000', 'labels': 'non-cj-change'}, {'hd_path': "m/84'/1'/2'/1/3", 'address': 'tb1qtqgvw445807tzcm8yhq6xgu3vmdfh66czx8jea', 'amount': '0.00800000', 'labels': 'non-cj-change'}, {'hd_path': "m/84'/1'/2'/1/4", 'address': 'tb1qxj7ulxdthe0dwxr5457p5d0w5u3jg7rwmc05pm', 'amount': '0.02400000', 'labels': 'non-cj-change'}, {'hd_path': "m/84'/1'/2'/1/5", 'address': 'tb1qv3kfe9ew42z0ldncgzmqcjznatsxz0vudvcjrv', 'amount': '0.00800000', 'labels': 'non-cj-change'}]}]}, {'account': '3', 'account_balance': '0.00000000', 'branches': [{'branch': "external addresses\tm/84'/1'/3'/0\ttpubDE9VN56aLW9BurCxHHGAWidSnVuU86ZsKPYQgxpTgkZxbogJYfj1vWJbtYip7WV5REcgmtjETb5eShXV8VUBzvCAMzuRm5Kv4ZGnnCiX6Jg", 'balance': '0.00000000', 'entries': [{'hd_path': "m/84'/1'/3'/0/0", 'address': 'tb1qp2w6ezmqn8nk9kc4gkpetgjj2mzqgp5x3hk86m', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/3'/0/1", 'address': 'tb1qd0tt93aulqs508mtap5p8gls5z57fqa4ggnfx7', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/3'/0/2", 'address': 'tb1qsp4hv46vgz4yjwt4p2wekh2gfmek7vgznrnd96', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/3'/0/3", 'address': 'tb1qvs322uyrwh7a74dsxel0xcrgucm27c6dzdmj9j', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/3'/0/4", 'address': 'tb1qnq9uk9azs9s7m5474ws7z7wxnwv3s3lxrtjter', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/3'/0/5", 'address': 'tb1q5tlq36q6ps0m9zu6h08gd3azsgkgvm73sjcmxw', 'amount': '0.00000000', 'labels': 'new'}]}, {'branch': "internal addresses\tm/84'/1'/3'/1\t", 'balance': '0.00000000', 'entries': []}]}, {'account': '4', 'account_balance': '0.00000000', 'branches': [{'branch': "external addresses\tm/84'/1'/4'/0\ttpubDE6QfTimeNgCFSYuxPPaLc1Cp3VokAuJAusYoiGwWtVHVtQDsepf5dRAFNLWMwpBCgKDYkXdWGs2JspxXPokrtooPh7db5fniqYbdKGqD4F", 'balance': '0.00000000', 'entries': [{'hd_path': "m/84'/1'/4'/0/3", 'address': 'tb1qr2llfup6cnh27n77nm7egcyf9r7c0ykucrcu8k', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/4'/0/4", 'address': 'tb1qahqjnd2y8j770l2m4kpf4fyfve9425c0zdumms', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/4'/0/5", 'address': 'tb1q0jm0cxwcm2g60489fvtmeeaf7mzg658t8f8fk4', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/4'/0/6", 'address': 'tb1qtpm5putpkzmrmecden0yytuuk4n9emhvxwqu8m', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/4'/0/7", 'address': 'tb1qn60fc04pmprn9wpzkt0dnt80awu0rpy99w376g', 'amount': '0.00000000', 'labels': 'new'}, {'hd_path': "m/84'/1'/4'/0/8", 'address': 'tb1qakvrpp2hd3a3303zx7w2shmvfc7tqk28pwa9sj', 'amount': '0.00000000', 'labels': 'new'}]}, {'branch': "internal addresses\tm/84'/1'/4'/1\t", 'balance': '0.00000000', 'entries': []}]}]} +``` \ No newline at end of file diff --git a/jmbase/jmbase/commands.py b/jmbase/jmbase/commands.py index 2d8eced..a937b7c 100644 --- a/jmbase/jmbase/commands.py +++ b/jmbase/jmbase/commands.py @@ -70,6 +70,13 @@ class JMMsgSignatureVerify(JMCommand): (b'fullmsg', Unicode()), (b'hostid', Unicode())] +class JMShutdown(JMCommand): + """ Requests shutdown of the current + message channel connections (to be used + when the client is shutting down). + """ + arguments = [] + """TAKER specific commands """ diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 5315029..11c6d68 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -31,7 +31,8 @@ from .blockchaininterface import (BlockchainInterface, from .snicker_receiver import SNICKERError, SNICKERReceiver from .client_protocol import (JMTakerClientProtocol, JMClientProtocolFactory, start_reactor, SNICKERClientProtocolFactory, - BIP78ClientProtocolFactory) + BIP78ClientProtocolFactory, + get_daemon_serving_params) from .podle import (set_commitment_file, get_commitment_file, add_external_commitments, PoDLE, generate_podle, get_podle_commitments, @@ -58,8 +59,10 @@ from .wallet_utils import ( wallet_change_passphrase) from .wallet_service import WalletService from .maker import Maker -from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain -from .payjoin import (parse_payjoin_setup, send_payjoin, +from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain, \ + YieldGeneratorService +from .snicker_receiver import SNICKERError, SNICKERReceiver, SNICKERReceiverService +from .payjoin import (parse_payjoin_setup, send_payjoin, PayjoinServer, JMBIP78ReceiverManager) # Set default logging handler to avoid "No handler found" warnings. diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 10062b9..cbdaff9 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -20,6 +20,15 @@ from jmclient import (jm_single, get_irc_mchannels, SNICKERReceiver, process_shutdown) import jmbitcoin as btc +# module level variable representing the port +# on which the daemon is running. +# note that this var is only set if we are running +# client+daemon in one process. +daemon_serving_port = -1 +daemon_serving_host = "" + +def get_daemon_serving_params(): + return (daemon_serving_host, daemon_serving_port) jlog = get_log() @@ -366,6 +375,15 @@ class JMClientProtocol(BaseClientProtocol): tx=tx) self.defaultCallbacks(d) + def request_mc_shutdown(self): + """ To ensure that lingering message channel + connections are shut down when the client itself + is shutting down. + """ + d = self.callRemote(commands.JMShutdown) + self.defaultCallbacks(d) + return {'accepted': True} + class JMMakerClientProtocol(JMClientProtocol): def __init__(self, factory, maker, nick_priv=None): self.factory = factory @@ -779,9 +797,9 @@ def start_reactor(host, port, factory=None, snickerfactory=None, #(Cannot start the reactor in tests) #Not used in prod (twisted logging): #startLogging(stdout) - usessl = True if jm_single().config.get("DAEMON", - "use_ssl") != 'false' else False - + global daemon_serving_host + global daemon_serving_port + usessl = True if jm_single().config.get("DAEMON", "use_ssl") != 'false' else False jmcport, snickerport, bip78port = [port]*3 if daemon: try: @@ -821,10 +839,13 @@ def start_reactor(host, port, factory=None, snickerfactory=None, p[0] += 1 return p[0] + if jm_coinjoin: # TODO either re-apply this port incrementing logic # to other protocols, or re-work how the ports work entirely. jmcport = start_daemon_on_port(port_a, dfactory, "Joinmarket", 0) + daemon_serving_port = jmcport + daemon_serving_host = host # (See above) For now these other two are just on ports that are 1K offsets. if snickerfactory: snickerport = start_daemon_on_port(port_a, sdfactory, "SNICKER", 1000) - 1000 @@ -840,17 +861,17 @@ def start_reactor(host, port, factory=None, snickerfactory=None, # Note the reactor.connect*** entries do not include BIP78 which # starts in jmclient.payjoin: - if usessl: - if factory: - reactor.connectSSL(host, jmcport, factory, ClientContextFactory()) - if snickerfactory: - reactor.connectSSL(host, snickerport, snickerfactory, - ClientContextFactory()) - else: - if factory: - reactor.connectTCP(host, jmcport, factory) - if snickerfactory: - reactor.connectTCP(host, snickerport, snickerfactory) + if usessl: + if factory: + reactor.connectSSL(host, jmcport, factory, ClientContextFactory()) + if snickerfactory: + reactor.connectSSL(host, snickerport, snickerfactory, + ClientContextFactory()) + else: + if factory: + reactor.connectTCP(host, jmcport, factory) + if snickerfactory: + reactor.connectTCP(host, snickerport, snickerfactory) if rs: if not gui: reactor.run(installSignalHandlers=ish) diff --git a/jmclient/jmclient/snicker_receiver.py b/jmclient/jmclient/snicker_receiver.py index cdc2f03..a17f406 100644 --- a/jmclient/jmclient/snicker_receiver.py +++ b/jmclient/jmclient/snicker_receiver.py @@ -1,5 +1,8 @@ #! /usr/bin/env python +import os +from twisted.application.service import Service +from twisted.internet import task import jmbitcoin as btc from jmclient.configure import jm_single from jmbase import (get_log, utxo_to_utxostr, @@ -11,7 +14,45 @@ jlog = get_log() class SNICKERError(Exception): pass -class SNICKERReceiver(Service): +class SNICKERReceiverService(Service): + def __init__(self, receiver): + assert isinstance(receiver, SNICKERReceiver) + self.receiver = receiver + # main monitor loop + self.monitor_loop = task.LoopingCall(self.receiver.poll_for_proposals) + + def startService(self): + """ Encapsulates start up actions. + This service depends on the receiver's + wallet service to start, so wait for that. + """ + self.wait_for_wallet = task.LoopingCall(self.wait_for_wallet_sync) + self.wait_for_wallet.start(5.0) + + def wait_for_wallet_sync(self): + if self.receiver.wallet_service.isRunning(): + jlog.info("SNICKER service starting because wallet service is up.") + self.wait_for_wallet.stop() + self.monitor_loop.start(5.0) + super().startService() + + def stopService(self, wallet=False): + """ Encapsulates shut down actions. + Optionally also shut down the underlying + wallet service (default False). + """ + if self.monitor_loop: + self.monitor_loop.stop() + if wallet: + self.receiver.wallet_service.stopService() + super().stopService() + + def isRunning(self): + if self.running == 1: + return True + return False + +class SNICKERReceiver(object): supported_flags = [] def __init__(self, wallet_service, acceptance_callback=None, @@ -66,6 +107,10 @@ class SNICKERReceiver(Service): def default_info_callback(self, msg): jlog.info(msg) + if not os.path.exists(self.proposals_source): + with open(self.proposals_source, "wb") as f: + jlog.info("created proposals source file.") + def default_acceptance_callback(self, our_ins, their_ins, our_outs, their_outs): diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 649541f..798c7b3 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -177,6 +177,12 @@ class WalletViewEntry(WalletViewBase): extradata = self.serialize_extra_data() return self.serclass(self.separator.join([left, addr, amounts, extradata])) + def serialize_json(self): + return {"hd_path": self.wallet_path_repr, + "address": self.serialize_address(), + "amount": self.serialize_amounts(), + "labels": self.serialize_extra_data()} + def serialize_wallet_position(self): return self.wallet_path_repr.ljust(20) @@ -229,6 +235,14 @@ class WalletViewBranch(WalletViewBase): lines.append(footer) return self.serclass(entryseparator.join(lines)) + def serialize_json(self, summarize=False): + if summarize: + return {} + else: + return {"branch": self.serialize_branch_header(), + "balance": self.get_fmt_balance(), + "entries": [x.serialize_json() for x in self.branchentries]} + def serialize_branch_header(self): start = "external addresses" if self.address_type == 0 else "internal addresses" if self.address_type == -1: @@ -263,6 +277,14 @@ class WalletViewAccount(WalletViewBase): return self.serclass(entryseparator.join([header] + [ x.serialize(entryseparator) for x in self.branches] + [footer])) + def serialize_json(self, summarize=False): + result = {"account": str(self.account), + "account_balance": self.get_fmt_balance()} + if summarize: + return result + result["branches"] = [x.serialize_json() for x in self.branches] + return result + class WalletView(WalletViewBase): def __init__(self, wallet_path_repr, accounts, wallet_name="JM wallet", serclass=str, custom_separator=None): @@ -286,6 +308,10 @@ class WalletView(WalletViewBase): return self.serclass(entryseparator.join([header] + [ x.serialize(entryseparator, summarize=False) for x in self.accounts] + [footer])) + def serialize_json(self, summarize=False): + return {"wallet_name": self.wallet_name, + "total_balance": self.get_fmt_balance(), + "accounts": [x.serialize_json(summarize=summarize) for x in self.accounts]} def get_tx_info(txid, tx_cache=None): """ @@ -393,7 +419,7 @@ def wallet_showutxos(wallet_service, showprivkey): def wallet_display(wallet_service, showprivkey, displayall=False, - serialized=True, summarized=False, mixdepth=None): + serialized=True, summarized=False, mixdepth=None, jsonified=False): """build the walletview object, then return its serialization directly if serialized, else return the WalletView object. @@ -546,7 +572,10 @@ def wallet_display(wallet_service, showprivkey, displayall=False, path = wallet_service.get_path_repr(wallet_service.get_path()) walletview = WalletView(path, acctlist) if serialized: - return walletview.serialize(summarize=summarized) + if jsonified: + return walletview.serialize_json(summarize=summarized) + else: + return walletview.serialize(summarize=summarized) else: return walletview diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index 4443765..a08c0c0 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -6,6 +6,7 @@ import time import abc import base64 from twisted.python.log import startLogging +from twisted.application.service import Service from optparse import OptionParser from jmbase import get_log from jmclient import (Maker, jm_single, load_program_config, @@ -263,6 +264,39 @@ class YieldGeneratorBasic(YieldGenerator): cjoutmix = (input_mixdepth + 1) % (self.wallet_service.mixdepth + 1) return self.wallet_service.get_internal_addr(cjoutmix) +class YieldGeneratorService(Service): + def __init__(self, wallet_service, daemon_host, daemon_port, yg_config): + self.wallet_service = wallet_service + self.daemon_host = daemon_host + self.daemon_port = daemon_port + self.yg_config = yg_config + self.yieldgen = None + + def startService(self): + """ We instantiate the Maker class only + here as its constructor will automatically + create orders based on the wallet. + Note makers already intrinsically handle + not-yet-synced wallet services, so there is + no need to check this here. + """ + # TODO genericise to any YG class: + self.yieldgen = YieldGeneratorBasic(self.wallet_service, self.yg_config) + self.clientfactory = JMClientProtocolFactory(self.yieldgen, proto_type="MAKER") + # here 'start_reactor' does not start the reactor but instantiates + # the connection to the daemon backend; note daemon=False, i.e. the daemon + # backend is assumed to be started elsewhere; we just connect to it with a client. + start_reactor(self.daemon_host, self.daemon_port, self.clientfactory, rs=False) + super().startService() + + def stopService(self): + """ TODO need a method exposed to gracefully + shut down a maker bot. + """ + if self.running: + jlog.info("Shutting down YieldGenerator service.") + self.clientfactory.proto_client.request_mc_shutdown() + super().stopService() def ygmain(ygclass, nickserv_password='', gaplimit=6): import sys diff --git a/jmdaemon/jmdaemon/daemon_protocol.py b/jmdaemon/jmdaemon/daemon_protocol.py index 2430cd3..97652f7 100644 --- a/jmdaemon/jmdaemon/daemon_protocol.py +++ b/jmdaemon/jmdaemon/daemon_protocol.py @@ -585,6 +585,12 @@ class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): self.mcc.on_verified_privmsg(nick, fullmsg, hostid) return {'accepted': True} + @JMShutdown.responder + def on_JM_SHUTDOWN(self): + self.mc_shutdown() + self.jm_state = 0 + return {'accepted': True} + """Taker specific responders """ diff --git a/jmdaemon/jmdaemon/irc.py b/jmdaemon/jmdaemon/irc.py index 25e25e6..3697b35 100644 --- a/jmdaemon/jmdaemon/irc.py +++ b/jmdaemon/jmdaemon/irc.py @@ -105,6 +105,10 @@ class IRCMessageChannel(MessageChannel): self.tx_irc_client = None #TODO can be configuration var, how long between reconnect attempts: self.reconnect_interval = 10 + + # service is used to wrap endpoints for Tor connections: + self.reconnecting_service = None + #implementation of abstract base class methods; #these are mostly but not exclusively acting as pass through #to the wrapped twisted IRC client protocol @@ -115,6 +119,8 @@ class IRCMessageChannel(MessageChannel): def shutdown(self): self.tx_irc_client.quit() self.give_up = True + if self.reconnecting_service: + self.reconnecting_service.stopService() def _pubmsg(self, msg): self.tx_irc_client._pubmsg(msg) @@ -157,8 +163,8 @@ class IRCMessageChannel(MessageChannel): use_tls = False ircEndpoint = TorSocksEndpoint(torEndpoint, self.serverport[0], self.serverport[1], tls=use_tls) - myRS = ClientService(ircEndpoint, factory) - myRS.startService() + self.reconnecting_service = ClientService(ircEndpoint, factory) + self.reconnecting_service.startService() else: try: factory = TxIRCFactory(self) @@ -203,7 +209,7 @@ class txIRC_Client(irc.IRCClient, object): def connectionLost(self, reason=protocol.connectionDone): wlog("INFO", "Lost IRC connection to: " + str(self.hostname) + " . Should reconnect automatically soon.") - if self.wrapper.on_disconnect: + if not self.wrapper.give_up and self.wrapper.on_disconnect: reactor.callLater(0.0, self.wrapper.on_disconnect, self.wrapper) return irc.IRCClient.connectionLost(self, reason) diff --git a/scripts/jmwalletd.py b/scripts/jmwalletd.py new file mode 100644 index 0000000..d7f1aa9 --- /dev/null +++ b/scripts/jmwalletd.py @@ -0,0 +1,420 @@ +#! /usr/bin/env python + +import datetime +import os +import time +import abc +import json +import atexit +from io import BytesIO +from twisted.python.log import startLogging +from twisted.internet import endpoints, reactor, ssl, task +from twisted.web.server import Site +from twisted.application.service import Service +from klein import Klein + +from optparse import OptionParser +from jmbase import get_log +from jmbitcoin import human_readable_transaction +from jmclient import Maker, jm_single, load_program_config, \ + JMClientProtocolFactory, start_reactor, calc_cj_fee, \ + WalletService, add_base_options, get_wallet_path, direct_send, \ + open_test_wallet_maybe, wallet_display, SegwitLegacyWallet, \ + SegwitWallet, get_daemon_serving_params, YieldGeneratorService, \ + SNICKERReceiverService, SNICKERReceiver, create_wallet, \ + StorageError, StoragePasswordError +from jmbase.support import EXIT_ARGERROR, EXIT_FAILURE + +jlog = get_log() + +# for debugging; twisted.web.server.Request objects do not easily serialize: +def print_req(request): + print(request) + print(request.method) + print(request.uri) + print(request.args) + print(request.path) + print(request.content) + print(list(request.requestHeaders.getAllRawHeaders())) + +class NotAuthorized(Exception): + pass + +class NoWalletFound(Exception): + pass + +class InvalidRequestFormat(Exception): + pass + +class BackendNotReady(Exception): + pass + +# error class for services which are only +# started once: +class ServiceAlreadyStarted(Exception): + pass + +# for the special case of the wallet service: +class WalletAlreadyUnlocked(Exception): + pass + +class ServiceNotStarted(Exception): + pass + +def get_ssl_context(cert_directory): + """Construct an SSL context factory from the user's privatekey/cert. + TODO: + Currently just hardcoded for tests. + """ + return ssl.DefaultOpenSSLContextFactory(os.path.join(cert_directory, "key.pem"), + os.path.join(cert_directory, "cert.pem")) + +def response(request, succeed=True, status=200, **kwargs): + """ + Build the response body as JSON and set the proper content-type + header. + """ + request.setHeader('Content-Type', 'application/json') + request.setHeader('Access-Control-Allow-Origin', '*') + request.setResponseCode(status) + return json.dumps( + [{'succeed': succeed, 'status': status, **kwargs}]) + +class JMWalletDaemon(Service): + """ This class functions as an HTTP/TLS server, + with acccess control, allowing a single client(user) + to control functioning of encapsulated Joinmarket services. + """ + + app = Klein() + def __init__(self, port): + """ Port is the port to serve this daemon + (using HTTP/TLS). + """ + # cookie tracks single user's state. + self.cookie = None + self.port = port + # the collection of services which this + # daemon may switch on and off: + self.services = {} + # master single wallet service which we + # allow the client to start/stop. + self.services["wallet"] = None + # label for convenience: + self.wallet_service = self.services["wallet"] + # Client may start other services, but only + # one instance. + self.services["snicker"] = None + self.services["maker"] = None + # ensure shut down does not leave dangling services: + atexit.register(self.stopService) + + def startService(self): + """ Encapsulates start up actions. + Here starting the TLS server. + """ + super().startService() + # we do not auto-start any service, including the base + # wallet service, since the client must actively request + # that with the appropriate credential (password). + reactor.listenSSL(self.port, Site(self.app.resource()), + contextFactory=get_ssl_context(".")) + + def stopService(self): + """ Encapsulates shut down actions. + """ + # Currently valid authorization tokens must be removed + # from the daemon: + self.cookie = None + # if the wallet-daemon is shut down, all services + # it encapsulates must also be shut down. + for name, service in self.services.items(): + if service: + service.stopService() + super().stopService() + + @app.handle_errors(NotAuthorized) + def not_authorized(self, request, failure): + request.setResponseCode(401) + return "Invalid credentials." + + @app.handle_errors(NoWalletFound) + def no_wallet_found(self, request, failure): + request.setResponseCode(404) + return "No wallet loaded." + + @app.handle_errors(BackendNotReady) + def backend_not_ready(self, request, failure): + request.setResponseCode(500) + return "Backend daemon not available" + + @app.handle_errors(InvalidRequestFormat) + def invalid_request_format(self, request, failure): + request.setResponseCode(401) + return "Invalid request format." + + @app.handle_errors(ServiceAlreadyStarted) + def service_already_started(self, request, failure): + request.setResponseCode(401) + return "Service already started." + + @app.handle_errors(WalletAlreadyUnlocked) + def wallet_already_unlocked(self, request, failure): + request.setResponseCode(401) + return "Wallet already unlocked." + + def service_not_started(self, request, failure): + request.setResponseCode(401) + return "Service cannot be stopped as it is not running." + + def check_cookie(self, request): + request_cookie = request.getHeader(b"JMCookie") + if self.cookie != request_cookie: + jlog.warn("Invalid cookie: " + str( + request_cookie) + ", request rejected.") + raise NotAuthorized() + + @app.route('/wallet//display', methods=['GET']) + def displaywallet(self, request, walletname): + print_req(request) + self.check_cookie(request) + if not self.wallet_service: + print("called display but no wallet loaded") + raise NoWalletFound() + else: + walletinfo = wallet_display(self.wallet_service, False, jsonified=True) + return response(request, walletname=walletname, walletinfo=walletinfo) + + # handling CORS preflight for any route: + @app.route('/', branch=True, methods=['OPTIONS']) + def preflight(self, request): + print_req(request) + request.setHeader("Access-Control-Allow-Origin", "*") + request.setHeader("Access-Control-Allow-Methods", "POST") + # "Cookie" is reserved so we specifically allow our custom cookie using + # name "JMCookie". + request.setHeader("Access-Control-Allow-Headers", "Content-Type, JMCookie") + + @app.route('/wallet//snicker/start', methods=['GET']) + def start_snicker(self, request, walletname): + self.check_cookie(request) + if not self.wallet_service: + raise NoWalletFound() + if self.services["snicker"] and self.services["snicker"].isRunning(): + raise ServiceAlreadyStarted() + # TODO: allow client to inject acceptance callbacks to Receiver + self.services["snicker"] = SNICKERReceiverService( + SNICKERReceiver(self.wallet_service)) + self.services["snicker"].startService() + # TODO waiting for startup seems perhaps not needed here? + return response(request, walletname=walletname) + + @app.route('/wallet//snicker/stop', methods=['GET']) + def stop_snicker(self, request, walletname): + self.check_cookie(request) + if not self.wallet_service: + raise NoWalletFound() + if not self.services["snicker"]: + raise ServiceNotStarted() + self.services["snicker"].stopService() + return response(request, walletname=walletname) + + @app.route('/wallet//taker/direct-send', methods=['POST']) + def send_direct(self, request, walletname): + """ Use the contents of the POST body to do a direct send from + the active wallet at the chosen mixdepth. + """ + assert isinstance(request.content, BytesIO) + payment_info_json = self.get_POST_body(request, ["mixdepth", "amount_sats", + "destination"]) + if not payment_info_json: + raise InvalidRequestFormat() + if not self.wallet_service: + raise NoWalletFound() + tx = direct_send(self.wallet_service, payment_info_json["amount_sats"], + payment_info_json["mixdepth"], + optin_rbf=payment_info_json["optin_rbf"], + return_transaction=True) + return response(request, walletname=walletname, + txinfo=human_readable_transaction(tx)) + + @app.route('/wallet//maker/start', methods=['POST']) + def start_maker(self, request, walletname): + """ Use the configuration in the POST body to start the yield generator: + """ + assert isinstance(request.content, BytesIO) + config_json = self.get_POST_body(request, ["txfee", "cjfee_a", "cjfee_r", + "ordertype", "minsize"]) + if not config_json: + raise InvalidRequestFormat() + if not self.wallet_service: + raise NoWalletFound() + + # daemon must be up before this is started; check: + daemon_serving_host, daemon_serving_port = get_daemon_serving_params() + if daemon_serving_port == -1 or daemon_serving_host == "": + raise BackendNotReady() + + self.services["maker"] = YieldGeneratorService(self.wallet_service, + daemon_serving_host, daemon_serving_port, + [config_json[x] for x in ["txfee", "cjfee_a", + "cjfee_r", "ordertype", "minsize"]]) + self.services["maker"].startService() + return response(request, walletname=walletname) + + @app.route('/wallet//maker/stop', methods=['GET']) + def stop_maker(self, request, walletname): + self.check_cookie(request) + if not self.wallet_service: + raise NoWalletFound() + if not self.services["maker"]: + raise ServiceNotStarted() + self.services["maker"].stopService() + return response(request, walletname=walletname) + + @app.route('/wallet//lock', methods=['GET']) + def lockwallet(self, request, walletname): + print_req(request) + self.check_cookie(request) + if not self.wallet_service: + print("called lock but no wallet loaded") + raise NoWalletFound() + else: + self.wallet_service.stopService() + self.wallet_service = None + # success status implicit: + return response(request, walletname=walletname) + + def get_POST_body(self, request, keys): + """ given a request object, retrieve values corresponding + to keys keys in a dict, assuming they were encoded using JSON. + If *any* of the keys are not present, return False, else + returns a dict of those key-value pairs. + """ + assert isinstance(request.content, BytesIO) + json_data = json.loads(request.content.read().decode("utf-8")) + retval = {} + for k in keys: + if k in json_data: + retval[k] = json_data[k] + else: + return False + return retval + + @app.route('/wallet/create', methods=["POST"]) + def createwallet(self, request): + print_req(request) + + # we only handle one wallet at a time; + # if there is a currently unlocked wallet, + # refuse to process the request: + if self.wallet_service: + raise WalletAlreadyUnlocked() + + request_data = self.get_POST_body(request, + ["walletname", "password", "wallettype"]) + if not request_data or request_data["wallettype"] not in [ + "sw", "sw-legacy"]: + raise InvalidRequestFormat() + + wallet_cls = SegwitWallet if request_data[ + "wallettype"]=="sw" else SegwitLegacyWallet + + # use the config's data location combined with the json + # data to construct the wallet path: + wallet_root_path = os.path.join(jm_single().datadir, "wallets") + wallet_name = os.path.join(wallet_root_path, request_data["walletname"]) + try: + wallet = create_wallet(wallet_name, request_data["password"], + 4, wallet_cls=wallet_cls) + except StorageError as e: + raise NotAuthorized(repr(e)) + + # finally, after the wallet is successfully created, we should + # start the wallet service: + return self.initialize_wallet_service(request, wallet) + + def initialize_wallet_service(self, request, wallet): + """ Called only when the wallet has loaded correctly, so + authorization is passed, so set cookie for this wallet + (currently THE wallet, daemon does not yet support multiple). + This is maintained for as long as the daemon is active (i.e. + no expiry currently implemented), or until the user switches + to a new wallet. + """ + self.cookie = request.getHeader(b"JMCookie") + if self.cookie is None: + raise NotAuthorized("No cookie") + + # the daemon blocks here until the wallet synchronization + # from the blockchain interface completes; currently this is + # fine as long as the client handles the response asynchronously: + self.wallet_service = WalletService(wallet) + while not self.wallet_service.synced: + self.wallet_service.sync_wallet(fast=True) + self.wallet_service.startService() + # now that the WalletService instance is active and ready to + # respond to requests, we return the status to the client: + return response(request, + walletname=self.wallet_service.get_wallet_name(), + already_loaded=False) + + @app.route('/wallet//unlock', methods=['POST']) + def unlockwallet(self, request, walletname): + print_req(request) + assert isinstance(request.content, BytesIO) + auth_json = self.get_POST_body(request, ["password"]) + if not auth_json: + raise InvalidRequestFormat() + password = auth_json["password"] + if self.wallet_service is None: + wallet_path = get_wallet_path(walletname, None) + try: + wallet = open_test_wallet_maybe( + wallet_path, walletname, 4, + password=password.encode("utf-8"), + ask_for_password=False) + except StoragePasswordError: + raise NotAuthorized("invalid password") + except StorageError as e: + # e.g. .lock file exists: + raise NotAuthorized(repr(e)) + return self.initialize_wallet_service(request, wallet) + else: + print('wallet was already unlocked.') + return response(request, + walletname=self.wallet_service.get_wallet_name(), + already_loaded=True) + +def jmwalletd_main(): + import sys + parser = OptionParser(usage='usage: %prog [options] [wallet file]') + parser.add_option('-p', '--port', action='store', type='int', + dest='port', default=28183, + help='the port over which to serve RPC, default 28183') + # TODO: remove the non-relevant base options: + add_base_options(parser) + + (options, args) = parser.parse_args() + + load_program_config(config_path=options.datadir) + + if jm_single().bc_interface is None: + jlog.error("Running jmwallet-daemon requires configured " + + "blockchain source.") + sys.exit(EXIT_FAILURE) + jlog.info("Starting jmwalletd on port: " + str(options.port)) + + jm_wallet_daemon = JMWalletDaemon(options.port) + jm_wallet_daemon.startService() + + nodaemon = jm_single().config.getint("DAEMON", "no_daemon") + daemon = True if nodaemon == 1 else False + if jm_single().config.get("BLOCKCHAIN", "network") in ["regtest", "testnet"]: + startLogging(sys.stdout) + start_reactor(jm_single().config.get("DAEMON", "daemon_host"), + jm_single().config.getint("DAEMON", "daemon_port"), + None, daemon=daemon) + +if __name__ == "__main__": + jmwalletd_main()