From 1688d2d2dc61d7e3d16a07f466e2e36696d6cd16 Mon Sep 17 00:00:00 2001 From: abhishek0405 Date: Sun, 8 Aug 2021 23:31:07 +0530 Subject: [PATCH] Adds listutxos and heartbeat route, several fixes The /utxos route is the equivalent of the showutxos wallet tool method. The heartbeat route /session allows a client to make sure the backend is still running and in the expected state (but see later commits for the coinjoin state update via the websocket). Fixes to sendpayment, the maker service, the create wallet function. sendpayment fix Also substantially improved and made functional the coinjoin route, a schedule is now created and a complete taker-side coinjoin is now possible. --- jmclient/jmclient/client_protocol.py | 12 +- jmclient/jmclient/taker_utils.py | 6 +- jmclient/jmclient/wallet_service.py | 2 +- jmclient/jmclient/yieldgenerator.py | 6 +- scripts/jmwalletd.py | 257 +++++++++++++++++++++++++-- 5 files changed, 257 insertions(+), 26 deletions(-) diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index cbdaff9..5a306c3 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -861,16 +861,16 @@ def start_reactor(host, port, factory=None, snickerfactory=None, # Note the reactor.connect*** entries do not include BIP78 which # starts in jmclient.payjoin: - if usessl: - if factory: + if usessl: + if factory: reactor.connectSSL(host, jmcport, factory, ClientContextFactory()) - if snickerfactory: + if snickerfactory: reactor.connectSSL(host, snickerport, snickerfactory, ClientContextFactory()) - else: - if factory: + else: + if factory: reactor.connectTCP(host, jmcport, factory) - if snickerfactory: + if snickerfactory: reactor.connectTCP(host, snickerport, snickerfactory) if rs: if not gui: diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index 867154c..a8d9767 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -21,7 +21,7 @@ Utility functions for tumbler-style takers; Currently re-used by CLI script tumbler.py and joinmarket-qt """ -def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, +def direct_send(wallet_service, amount, mixdepth, destination, answeryes=True, accept_callback=None, info_callback=None, error_callback=None, return_transaction=False, with_final_psbt=False, optin_rbf=False, custom_change_addr=None): @@ -189,15 +189,19 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info(sending_info) if not answeryes: if not accept_callback: + if input('Would you like to push to the network? (y/n):')[0] != 'y': + log.info("You chose not to broadcast the transaction, quitting.") return False else: + accepted = accept_callback(human_readable_transaction(tx), destination, actual_amount, fee_est, custom_change_addr) if not accepted: return False + print("here is ",jm_single().bc_interface.pushtx(tx.serialize())) if jm_single().bc_interface.pushtx(tx.serialize()): txid = bintohex(tx.GetTxid()[::-1]) successmsg = "Transaction sent: " + txid diff --git a/jmclient/jmclient/wallet_service.py b/jmclient/jmclient/wallet_service.py index 22046b1..a390071 100644 --- a/jmclient/jmclient/wallet_service.py +++ b/jmclient/jmclient/wallet_service.py @@ -714,7 +714,7 @@ class WalletService(Service): import_needed = self.bci.import_addresses_if_needed(addresses, wallet_name) if import_needed: - self.display_rescan_message_and_system_exit(self.restart_callback) + #self.display_rescan_message_and_system_exit(self.restart_callback) return if isinstance(self.wallet, FidelityBondMixin): diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index a08c0c0..638ab47 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -82,11 +82,14 @@ class YieldGeneratorBasic(YieldGenerator): thus is somewhat suboptimal in giving more information to spies. """ def __init__(self, wallet_service, offerconfig): - # note the randomizing entries are ignored in this base class: + #note the randomizing entries are ignored in this base class: + self.txfee, self.cjfee_a, self.cjfee_r, self.ordertype, self.minsize, \ self.txfee_factor, self.cjfee_factor, self.size_factor = offerconfig super().__init__(wallet_service) + + def create_my_orders(self): mix_balance = self.get_available_mixdepths() if len([b for m, b in mix_balance.items() if b > 0]) == 0: @@ -295,6 +298,7 @@ class YieldGeneratorService(Service): """ if self.running: jlog.info("Shutting down YieldGenerator service.") + print("client fac is ",self.clientfactory) self.clientfactory.proto_client.request_mc_shutdown() super().stopService() diff --git a/scripts/jmwalletd.py b/scripts/jmwalletd.py index d7f1aa9..289a1be 100644 --- a/scripts/jmwalletd.py +++ b/scripts/jmwalletd.py @@ -1,5 +1,6 @@ #! /usr/bin/env python +from jmbitcoin import * import datetime import os import time @@ -7,6 +8,7 @@ import abc import json import atexit from io import BytesIO +from jmclient.wallet_utils import wallet_showseed,wallet_showutxos from twisted.python.log import startLogging from twisted.internet import endpoints, reactor, ssl, task from twisted.web.server import Site @@ -16,17 +18,21 @@ from klein import Klein from optparse import OptionParser from jmbase import get_log from jmbitcoin import human_readable_transaction -from jmclient import Maker, jm_single, load_program_config, \ +from jmclient import Taker, Maker, jm_single, load_program_config, \ JMClientProtocolFactory, start_reactor, calc_cj_fee, \ WalletService, add_base_options, get_wallet_path, direct_send, \ - open_test_wallet_maybe, wallet_display, SegwitLegacyWallet, \ + open_test_wallet_maybe, wallet, wallet_display, SegwitLegacyWallet, \ SegwitWallet, get_daemon_serving_params, YieldGeneratorService, \ SNICKERReceiverService, SNICKERReceiver, create_wallet, \ - StorageError, StoragePasswordError -from jmbase.support import EXIT_ARGERROR, EXIT_FAILURE + StorageError, StoragePasswordError, get_max_cj_fee_values +from jmbase.support import get_log, set_logging_level, jmprint,EXIT_ARGERROR, EXIT_FAILURE,DUST_THRESHOLD +import glob + +import jwt jlog = get_log() + # for debugging; twisted.web.server.Request objects do not easily serialize: def print_req(request): print(request) @@ -91,6 +97,7 @@ class JMWalletDaemon(Service): """ Port is the port to serve this daemon (using HTTP/TLS). """ + print("in init") # cookie tracks single user's state. self.cookie = None self.port = port @@ -167,9 +174,24 @@ class JMWalletDaemon(Service): request.setResponseCode(401) return "Service cannot be stopped as it is not running." + # def check_cookie(self, request): + # request_cookie = request.getHeader(b"JMCookie") + # if self.cookie != request_cookie: + # jlog.warn("Invalid cookie: " + str( + # request_cookie) + ", request rejected.") + # raise NotAuthorized() + def check_cookie(self, request): - request_cookie = request.getHeader(b"JMCookie") - if self.cookie != request_cookie: + print("header details:") + #part after bearer is what we need + auth_header=((request.getHeader('Authorization'))) + request_cookie = None + if auth_header is not None: + request_cookie=auth_header[7:] + + print("request cookie is",request_cookie) + print("actual cookie is",self.cookie) + if request_cookie==None or self.cookie != request_cookie: jlog.warn("Invalid cookie: " + str( request_cookie) + ", request rejected.") raise NotAuthorized() @@ -185,6 +207,16 @@ class JMWalletDaemon(Service): walletinfo = wallet_display(self.wallet_service, False, jsonified=True) return response(request, walletname=walletname, walletinfo=walletinfo) + #Heartbeat route + + @app.route('/session',methods=['GET']) + def sessionExists(self, request): + #if no wallet loaded then clear frontend session info + #when no wallet status is false + session = not self.cookie==None + return response(request,session=session) + + # handling CORS preflight for any route: @app.route('/', branch=True, methods=['OPTIONS']) def preflight(self, request): @@ -224,17 +256,26 @@ class JMWalletDaemon(Service): """ Use the contents of the POST body to do a direct send from the active wallet at the chosen mixdepth. """ + self.check_cookie(request) assert isinstance(request.content, BytesIO) + payment_info_json = self.get_POST_body(request, ["mixdepth", "amount_sats", "destination"]) + if not payment_info_json: raise InvalidRequestFormat() if not self.wallet_service: raise NoWalletFound() - tx = direct_send(self.wallet_service, payment_info_json["amount_sats"], - payment_info_json["mixdepth"], - optin_rbf=payment_info_json["optin_rbf"], - return_transaction=True) + + tx = direct_send(self.wallet_service, int(payment_info_json["amount_sats"]), + int(payment_info_json["mixdepth"]), + destination=payment_info_json["destination"], + return_transaction=True,answeryes=True) + + # tx = direct_send(self.wallet_service, payment_info_json["amount_sats"], + # payment_info_json["mixdepth"], + # optin_rbf=payment_info_json["optin_rbf"], + # return_transaction=True) return response(request, walletname=walletname, txinfo=human_readable_transaction(tx)) @@ -242,6 +283,7 @@ class JMWalletDaemon(Service): def start_maker(self, request, walletname): """ Use the configuration in the POST body to start the yield generator: """ + self.check_cookie(request) assert isinstance(request.content, BytesIO) config_json = self.get_POST_body(request, ["txfee", "cjfee_a", "cjfee_r", "ordertype", "minsize"]) @@ -255,10 +297,21 @@ class JMWalletDaemon(Service): if daemon_serving_port == -1 or daemon_serving_host == "": raise BackendNotReady() + for key,val in config_json.items(): + if(key == 'cjfee_r' or key == 'ordertype'): + pass + + else: + config_json[key] = int(config_json[key]) +# self.txfee_factor, self.cjfee_factor, self.size_factor + config_json['txfee_factor'] = None + config_json["cjfee_factor"] = None + config_json["size_factor"] = None + self.services["maker"] = YieldGeneratorService(self.wallet_service, daemon_serving_host, daemon_serving_port, [config_json[x] for x in ["txfee", "cjfee_a", - "cjfee_r", "ordertype", "minsize"]]) + "cjfee_r", "ordertype", "minsize","txfee_factor","cjfee_factor","size_factor"]]) self.services["maker"].startService() return response(request, walletname=walletname) @@ -281,6 +334,7 @@ class JMWalletDaemon(Service): raise NoWalletFound() else: self.wallet_service.stopService() + self.cookie = None self.wallet_service = None # success status implicit: return response(request, walletname=walletname) @@ -313,6 +367,8 @@ class JMWalletDaemon(Service): request_data = self.get_POST_body(request, ["walletname", "password", "wallettype"]) + + if not request_data or request_data["wallettype"] not in [ "sw", "sw-legacy"]: raise InvalidRequestFormat() @@ -324,17 +380,25 @@ class JMWalletDaemon(Service): # data to construct the wallet path: wallet_root_path = os.path.join(jm_single().datadir, "wallets") wallet_name = os.path.join(wallet_root_path, request_data["walletname"]) + try: - wallet = create_wallet(wallet_name, request_data["password"], + wallet = create_wallet(wallet_name, request_data["password"].encode("ascii"), 4, wallet_cls=wallet_cls) + print("seedphrase is ") + seedphrase_help_string = wallet_showseed(wallet) + + except StorageError as e: raise NotAuthorized(repr(e)) # finally, after the wallet is successfully created, we should # start the wallet service: - return self.initialize_wallet_service(request, wallet) - def initialize_wallet_service(self, request, wallet): + #return response(request,message="Wallet Created Succesfully,unlock it for further use") + return self.initialize_wallet_service(request, wallet, seedphrase=seedphrase_help_string) + + + def initialize_wallet_service(self, request, wallet,**kwargs): """ Called only when the wallet has loaded correctly, so authorization is passed, so set cookie for this wallet (currently THE wallet, daemon does not yet support multiple). @@ -342,7 +406,18 @@ class JMWalletDaemon(Service): no expiry currently implemented), or until the user switches to a new wallet. """ - self.cookie = request.getHeader(b"JMCookie") + + encoded_token = jwt.encode({"wallet": "name_of_wallet","exp" :datetime.datetime.utcnow()+datetime.timedelta(minutes=30)},"secret") + encoded_token = encoded_token.strip() + print(encoded_token) + # decoded_token = jwt.decode(encoded_token,"secret",algorithms=["HS256"]) + # print(decoded_token) + # request.addCookie(b'session_token', encoded_token) + # self.cookie = encoded_token + self.cookie = encoded_token + #self.cookie = request.getHeader(b"JMCookie") + + if self.cookie is None: raise NotAuthorized("No cookie") @@ -355,13 +430,21 @@ class JMWalletDaemon(Service): self.wallet_service.startService() # now that the WalletService instance is active and ready to # respond to requests, we return the status to the client: - return response(request, + + #def response(request, succeed=True, status=200, **kwargs): + if('seedphrase' in kwargs): + return response(request, walletname=self.wallet_service.get_wallet_name(), - already_loaded=False) + already_loaded=False,token=encoded_token,seedphrase = kwargs.get('seedphrase')) + else: + return response(request, + walletname=self.wallet_service.get_wallet_name(), + already_loaded=False,token=encoded_token) @app.route('/wallet//unlock', methods=['POST']) def unlockwallet(self, request, walletname): print_req(request) + #print(get_current_chain_params()) assert isinstance(request.content, BytesIO) auth_json = self.get_POST_body(request, ["password"]) if not auth_json: @@ -386,6 +469,144 @@ class JMWalletDaemon(Service): walletname=self.wallet_service.get_wallet_name(), already_loaded=True) + + #This route should return list of current wallets created. + @app.route('/wallet/all', methods=['GET']) + def listwallets(self, request): + #this is according to the assumption that wallets are there in /.joinmarket by default, also currently path for linux system only. + #first user taken for path + user_path = glob.glob('/home/*/')[0] + + wallet_dir = f"{user_path}.joinmarket/wallets/*.jmdat" + wallets = (glob.glob(wallet_dir)) + + offset = len(user_path)+len('.joinmarket/wallets/') + #to get only names + short_wallets = [wallet[offset:] for wallet in wallets] + return response(request,wallets=short_wallets) + + #route to get external address for deposit + @app.route('/address/new/',methods=['GET']) + def getaddress(self, request, mixdepth): + + self.check_cookie(request) + if not self.wallet_service: + raise NoWalletFound() + mixdepth = int(mixdepth) + address = self.wallet_service.get_external_addr(mixdepth) + return response(request,address=address) + + #route to list utxos + @app.route('/wallet/utxos',methods=['GET']) + def listUtxos(self, request): + self.check_cookie(request) + if not self.wallet_service: + raise NoWalletFound() + utxos = wallet_showutxos(self.wallet_service, False) + + return response(request,transactions=utxos) + + #return True for now + def filter_orders_callback(self,orderfees, cjamount): + return True + + + #route to start a coinjoin transaction + @app.route('/wallet/taker/coinjoin',methods=['POST']) + def doCoinjoin(self, request): + self.check_cookie(request) + if not self.wallet_service: + raise NoWalletFound() + + request_data = self.get_POST_body(request,["mixdepth", "amount", "counterparties","destination"]) + #refer sample schedule testnet + waittime = 0 + rounding=16 + completion_flag=0 + #list of list + schedule = [[int(request_data["mixdepth"]), int(request_data["amount"]), int(request_data["counterparties"]), request_data["destination"], waittime, rounding, completion_flag]] + print(schedule) + #instantiate a taker + #keeping order_chooser as default for now + + #max_cj_feee is to be set based on config values (jmsingle.config.get policy var->max cj fee abs in configure.py) + + max_cj_fee=(1,float('inf')) + print("max cj fee is,",max_cj_fee) + self.taker = Taker(self.wallet_service, schedule, max_cj_fee = max_cj_fee, callbacks=(self.filter_orders_callback, None, self.taker_finished)) + + clientfactory = JMClientProtocolFactory(self.taker) + + nodaemon = jm_single().config.getint("DAEMON", "no_daemon") + daemon = True if nodaemon == 1 else False + dhost = jm_single().config.get("DAEMON", "daemon_host") + dport = jm_single().config.getint("DAEMON", "daemon_port") + + if jm_single().config.get("BLOCKCHAIN", "network") == "regtest": + startLogging(sys.stdout) + start_reactor(dhost, dport, clientfactory, daemon=daemon, rs=False) + + def taker_finished(self, res, fromtx=False, waittime=0.0, txdetails=None): + + if fromtx == "unconfirmed": + #If final entry, stop *here*, don't wait for confirmation + return + if fromtx: + if res: + txd, txid = txdetails + reactor.callLater(waittime*60, + clientfactory.getClient().clientStart) + else: + #a transaction failed; we'll try to repeat without the + #troublemakers. + #If this error condition is reached from Phase 1 processing, + #and there are less than minimum_makers honest responses, we + #just give up (note that in tumbler we tweak and retry, but + #for sendpayment the user is "online" and so can manually + #try again). + #However if the error is in Phase 2 and we have minimum_makers + #or more responses, we do try to restart with the honest set, here. + if self.taker.latest_tx is None: + #can only happen with < minimum_makers; see above. + jlog.info("A transaction failed but there are insufficient " + "honest respondants to continue; giving up.") + reactor.stop() + return + #This is Phase 2; do we have enough to try again? + self.taker.add_honest_makers(list(set( + self.taker.maker_utxo_data.keys()).symmetric_difference( + set(self.taker.nonrespondants)))) + if len(self.taker.honest_makers) < jm_single().config.getint( + "POLICY", "minimum_makers"): + jlog.info("Too few makers responded honestly; " + "giving up this attempt.") + reactor.stop() + return + jmprint("We failed to complete the transaction. The following " + "makers responded honestly: " + str(self.taker.honest_makers) +\ + ", so we will retry with them.", "warning") + #Now we have to set the specific group we want to use, and hopefully + #they will respond again as they showed honesty last time. + #we must reset the number of counterparties, as well as fix who they + #are; this is because the number is used to e.g. calculate fees. + #cleanest way is to reset the number in the schedule before restart. + self.taker.schedule[self.taker.schedule_index][2] = len(self.taker.honest_makers) + jlog.info("Retrying with: " + str(self.taker.schedule[ + self.taker.schedule_index][2]) + " counterparties.") + #rewind to try again (index is incremented in Taker.initialize()) + self.taker.schedule_index -= 1 + self.taker.set_honest_only(True) + reactor.callLater(5.0, clientfactory.getClient().clientStart) + else: + if not res: + jlog.info("Did not complete successfully, shutting down") + #Should usually be unreachable, unless conf received out of order; + #because we should stop on 'unconfirmed' for last (see above) + else: + jlog.info("All transactions completed correctly") + reactor.stop() + + def jmwalletd_main(): import sys parser = OptionParser(usage='usage: %prog [options] [wallet file]') @@ -416,5 +637,7 @@ def jmwalletd_main(): jm_single().config.getint("DAEMON", "daemon_port"), None, daemon=daemon) + + if __name__ == "__main__": jmwalletd_main()