RPC-API: add ability to recover wallet

Fixes #1082:

This commit allows recovery of a wallet from a seedphrase
with a new endpoint wallet/recover. 4 parameters are passed in,
the same three as for wallet/create but also a bip39 seedphrase
as the fourth argument.

This commit also adds a rescanblockchain RPC call and status:

This adds a new endpoint /rescanblockchain which is (Core) wallet specific
(due to underlying JM architecture). This action is spawned in a thread,
since the bitcoind RPC call is blocking and can take a very long time.
To monitor the status of the rescan, an extra field `rescanning` is
added to the /session endpoint.

Also adds test of rpc wallet recovery

docs/api/wallet-rpc.yaml

@@ -30,6 +30,26 @@ paths:
             schema:
               $ref: '#/components/schemas/CreateWalletRequest'
         description: wallet creation parameters
+  /wallet/recover:
+    post:
+      summary: recover a wallet from a seedphrase
+      operationId: recoverwallet
+      description: Give a filename (.jmdat must be included), a wallettype, a seedphrase and a password, create the wallet for the newly persisted wallet file. The wallettype variable must be one of "sw" - segwit native, "sw-legacy" - segwit legacy or "sw-fb" - segwit native with fidelity bonds supported, the last of which is the default. The seedphrase must be a single string with words space-separated, and must conform to BIP39 (else 400 is returned). Note that this operation cannot be performed when a wallet is already loaded (unlocked).
+      responses:
+        '201':
+          $ref: '#/components/responses/Create-201-OK'
+        '400':
+          $ref: '#/components/responses/400-BadRequest'
+        '401':
+          $ref: '#/components/responses/401-Unauthorized'
+        '409':
+          $ref: '#/components/responses/409-AlreadyExists'
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/RecoverWalletRequest'
+        description: wallet recovery parameters
   /wallet/{walletname}/unlock:
     post:
       summary: decrypt an existing wallet
@@ -171,6 +191,33 @@ paths:
           $ref: '#/components/responses/401-Unauthorized'
         '404':
           $ref: '#/components/responses/404-NotFound'
+  /wallet/{walletname}/rescanblockchain/{blockheight}:
+    get:
+      summary: Rescan the blockchain from a given blockheight
+      operationId: rescanblockchain
+      description: Use this operation on recovered wallets to re-sync the wallet
+      parameters:
+        - name: walletname
+          in: path
+          description: name of wallet including .jmdat
+          required: true
+          schema:
+            type: string
+        - name: blockheight
+          in: path
+          description: starting block height for the rescan
+          required: true
+          schema:
+            type: integer
+      responses:
+        '200':
+          $ref: "#/components/responses/RescanBlockchain-200-OK"
+        '400':
+          $ref: '#/components/responses/400-BadRequest'
+        '401':
+          $ref: '#/components/responses/401-Unauthorized'
+        '404':
+          $ref: '#/components/responses/404-NotFound'
   /wallet/{walletname}/address/timelock/new/{lockdate}:
     get:
       security:
@@ -714,6 +761,7 @@ components:
         - maker_running
         - coinjoin_in_process
         - wallet_name
+        - rescanning
       properties:
         session:
           type: boolean
@@ -751,6 +799,8 @@ components:
                 type: string
         nickname:
           type: string
+        rescanning:
+          type: boolean
     ListUtxosResponse:
       type: object
       properties:
@@ -960,6 +1010,27 @@ components:
         wallettype:
           type: string
           example: "sw-fb"
+    RecoverWalletRequest:
+      type: object
+      required:
+        - walletname
+        - password
+        - wallettype
+        - seedphrase
+      properties:
+        walletname:
+          type: string
+          example: wallet.jmdat
+        password:
+          type: string
+          format: password
+          example: hunter2
+        wallettype:
+          type: string
+          example: "sw-fb"
+        seedphrase:
+          type: string
+          example: "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about"
     UnlockWalletRequest:
       type: object
       required:
@@ -1035,12 +1106,24 @@ components:
         application/json:
           schema:
             $ref: "#/components/schemas/SessionResponse"
+    RescanBlockchain-200-OK:
+      description: "Blockchain rescan started successfully"
+      content:
+        application/json:
+          schema:
+            $ref: "#/components/schemas/SessionResponse"
     Create-201-OK:
       description: "wallet created successfully"
       content:
         application/json:
           schema:
             $ref: "#/components/schemas/CreateWalletResponse"
+    Recover-201-OK:
+      description: "wallet recovered successfully"
+      content:
+        application/json:
+          schema:
+            $ref: "#/components/schemas/CreateWalletResponse"
     Unlock-200-OK:
       description: "wallet unlocked successfully"
       content:

jmclient/jmclient/blockchaininterface.py

@@ -4,6 +4,7 @@ import ast
 import random
 import sys
 import time
+from typing import Optional
 from decimal import Decimal
 import binascii
 from twisted.internet import reactor, task
@@ -215,6 +216,39 @@ class BitcoinCoreInterface(BlockchainInterface):
             return False
         return block
 
+    def rescanblockchain(self, start_height: int, end_height: Optional[int] = None) -> None:
+        # Threading is not used in Joinmarket but due to blocking
+        # nature of this very slow RPC call, we need to fire and forget.
+        from threading import Thread
+        Thread(target=self.rescan_in_thread, args=(start_height,),
+               daemon=True).start()
+
+    def rescan_in_thread(self, start_height: int) -> None:
+        """ In order to not conflict with the existing main
+        JsonRPC connection in the main thread, this rescanning
+        thread creates a distinct JsonRPC object, just to make
+        this one RPC call `rescanblockchain <height>`, using the
+        same credentials.
+        """
+        from jmclient.jsonrpc import JsonRpc
+        authstr = self.jsonRpc.authstr
+        user, password = authstr.split(":")
+        newjsonRpc = JsonRpc(self.jsonRpc.host,
+                             self.jsonRpc.port,
+                             user, password,
+                             url=self.jsonRpc.url)
+        try:
+            newjsonRpc.call('rescanblockchain', [start_height])
+        except JsonRpcConnectionError:
+            log.error("Failure of RPC connection to Bitcoin Core. "
+                      "Rescanning process not started.")
+
+    def getwalletinfo(self) -> dict:
+        """ Returns detailed about currently loaded (see `loadwallet`
+        call in __init__) Bitcoin Core wallet.
+        """
+        return self._rpc("getwalletinfo", [])
+
     def _rpc(self, method, args):
         """ Returns the result of an rpc call to the Bitcoin Core RPC API.
         If the connection is permanently or unrecognizably broken, None

jmclient/jmclient/wallet_rpc.py

@@ -25,8 +25,8 @@ from jmclient import Taker, jm_single, \
     NotEnoughFundsException, get_tumble_log, get_tumble_schedule, \
     get_schedule, get_tumbler_parser, schedule_to_text, \
     tumbler_filter_orders_callback, tumbler_taker_finished_update, \
-    validate_address, FidelityBondMixin, \
+    validate_address, FidelityBondMixin, BaseWallet, WalletError, \
-    ScheduleGenerationErrorNoFunds
+    ScheduleGenerationErrorNoFunds, BIP39WalletMixin
 from jmbase.support import get_log, utxostr_to_utxo
 
 jlog = get_log()
@@ -108,6 +108,7 @@ class NotEnoughCoinsForTumbler(Exception):
 class YieldGeneratorDataUnreadable(Exception):
     pass
 
+
 def get_ssl_context(cert_directory):
     """Construct an SSL context factory from the user's privatekey/cert.
     TODO:
@@ -476,6 +477,8 @@ class JMWalletDaemon(Service):
 
         # First, prepare authentication for the calling client:
         self.set_token(wallet_name)
+        # return type is different for a newly created OR recovered
+        # wallet, in this case we use the 'seedphrase' kwarg as trigger:
         if('seedphrase' in kwargs):
             return make_jmwalletd_response(request,
                         status=201,
@@ -551,6 +554,23 @@ class JMWalletDaemon(Service):
             raise BackendNotReady()
         return (daemon_serving_host, daemon_serving_port)
 
+    def get_wallet_cls_from_type(self, wallettype: str) -> BaseWallet:
+        if wallettype == "sw":
+            return SegwitWallet
+        elif wallettype == "sw-legacy":
+            return SegwitLegacyWallet
+        elif wallettype == "sw-fb":
+            return SegwitWalletFidelityBonds
+        else:
+            raise InvalidRequestFormat()
+
+    def get_wallet_name_from_req(self, walletname: str) -> str:
+        """ use the config's data location combined with the json
+        data from the request to construct the wallet path
+        """
+        wallet_root_path = os.path.join(jm_single().datadir, "wallets")
+        return os.path.join(wallet_root_path, walletname)
+
     """ RPC begins here.
     """
 
@@ -576,6 +596,25 @@ class JMWalletDaemon(Service):
                 walletinfo = wallet_display(self.services["wallet"], False, jsonified=True)
                 return make_jmwalletd_response(request, walletname=walletname, walletinfo=walletinfo)
 
+        @app.route('/wallet/<string:walletname>/rescanblockchain/<int:blockheight>', methods=['GET'])
+        def rescanblockchain(self, request, walletname, blockheight):
+            """ This route lets the user trigger the rescan action in the backend.
+            Note that it technically "shouldn't" require a wallet to be loaded,
+            but since we hide all blockchain access behind the wallet service,
+            it currently *does* require this.
+            """
+            print_req(request)
+            self.check_cookie(request)
+            if not self.services["wallet"]:
+                jlog.warn("rescanblockchain called, but no wallet service active.")
+                raise NoWalletFound()
+            if not self.wallet_name == walletname:
+                jlog.warn("called rescanblockchain with wrong wallet")
+                raise InvalidRequestFormat()
+            else:
+                self.services["wallet"].rescanblockchain(blockheight)
+                return make_jmwalletd_response(request, walletname=walletname)
+
         @app.route('/session', methods=['GET'])
         def session(self, request):
             """ This route functions as a heartbeat, and communicates
@@ -596,9 +635,17 @@ class JMWalletDaemon(Service):
             schedule = None
             offer_list = None
             nickname = None
+            # We don't technically *know* the backend is not
+            # rescanning, but that would be a strange scenario:
+            rescanning = False
 
             if self.services["wallet"]:
                 if self.services["wallet"].isRunning():
+                    winfo = self.services["wallet"].get_backend_walletinfo()
+                    if "scanning" in winfo and winfo["scanning"]:
+                        # Note that if not 'false', it contains info
+                        # that looks like: {'duration': 1, 'progress': Decimal('0.04665404082350701')}
+                        rescanning = True
                     wallet_name = self.wallet_name
                     # At this point if an `auth_header` is present, it has been checked
                     # by the call to `check_cookie_if_present` above.
@@ -626,7 +673,8 @@ class JMWalletDaemon(Service):
                             schedule=schedule,
                             wallet_name=wallet_name,
                             offer_list=offer_list,
-                            nickname=nickname)
+                            nickname=nickname,
+                            rescanning=rescanning)
 
         @app.route('/wallet/<string:walletname>/taker/direct-send', methods=['POST'])
         def directsend(self, request, walletname):
@@ -829,22 +877,11 @@ class JMWalletDaemon(Service):
                             ["walletname", "password", "wallettype"])
             if not request_data:
                 raise InvalidRequestFormat()
-            wallettype = request_data["wallettype"]
+            wallet_cls = self.get_wallet_cls_from_type(
-            if wallettype == "sw":
+                request_data["wallettype"])
-                wallet_cls = SegwitWallet
-            elif wallettype == "sw-legacy":
-                wallet_cls = SegwitLegacyWallet
-            elif wallettype == "sw-fb":
-                wallet_cls = SegwitWalletFidelityBonds
-            else:
-                raise InvalidRequestFormat()
-            # use the config's data location combined with the json
-            # data to construct the wallet path:
-            wallet_root_path = os.path.join(jm_single().datadir, "wallets")
-            wallet_name = os.path.join(wallet_root_path,
-                                       request_data["walletname"])
             try:
-                wallet = create_wallet(wallet_name,
+                wallet = create_wallet(self.get_wallet_name_from_req(
+                    request_data["walletname"]),
                     request_data["password"].encode("ascii"),
                     4, wallet_cls=wallet_cls)
                 # extension not yet supported in RPC create; TODO
@@ -859,6 +896,45 @@ class JMWalletDaemon(Service):
                                         request_data["walletname"],
                                         seedphrase=seed)
 
+        @app.route('/wallet/recover', methods=["POST"])
+        def recoverwallet(self, request):
+            print_req(request)
+            # we only handle one wallet at a time;
+            # if there is a currently unlocked wallet,
+            # refuse to process the request:
+            if self.services["wallet"]:
+                raise WalletAlreadyUnlocked()
+            request_data = self.get_POST_body(request,
+                            ["walletname", "password",
+                             "wallettype", "seedphrase"])
+            if not request_data:
+                raise InvalidRequestFormat()
+            wallet_cls = self.get_wallet_cls_from_type(
+                request_data["wallettype"])
+            seedphrase = request_data["seedphrase"]
+            seedphrase = seedphrase.strip()
+            if not seedphrase:
+                raise InvalidRequestFormat()
+            try:
+                entropy = BIP39WalletMixin.entropy_from_mnemonic(seedphrase)
+            except WalletError:
+                # should only occur if the seedphrase is not valid BIP39:
+                raise InvalidRequestFormat()
+            try:
+                wallet = create_wallet(self.get_wallet_name_from_req(
+                    request_data["walletname"]),
+                    request_data["password"].encode("ascii"),
+                    4, wallet_cls=wallet_cls, entropy=entropy)
+            except RetryableStorageError:
+                raise LockExists()
+            except StorageError:
+                raise WalletAlreadyExists()
+            # finally, after the wallet is successfully created, we should
+            # start the wallet service, then return info to the caller:
+            return self.initialize_wallet_service(request, wallet,
+                                        request_data["walletname"],
+                                        seedphrase=seedphrase)
+
         @app.route('/wallet/<string:walletname>/unlock', methods=['POST'])
         def unlockwallet(self, request, walletname):
             """ If a user succeeds in authenticating and opening a

jmclient/jmclient/wallet_service.py

@@ -4,6 +4,7 @@ import collections
 import itertools
 import time
 import sys
+from typing import Optional
 from decimal import Decimal
 from copy import deepcopy
 from twisted.internet import reactor
@@ -729,6 +730,15 @@ class WalletService(Service):
     def get_block_height(self, blockhash):
         return self.bci.get_block_height(blockhash)
 
+    def rescanblockchain(self, start_height: int, end_height: Optional[int] = None) -> None:
+        self.bci.rescanblockchain(start_height, end_height)
+
+    def get_backend_walletinfo(self) -> dict:
+        """ 'Backend' wallet means the Bitcoin Core wallet,
+        which will always be loaded if self.bci is init-ed.
+        """
+        return self.bci.getwalletinfo()
+
     def get_transaction_block_height(self, tx):
         """ Given a CTransaction object tx, return
         the block height at which it was mined, or False

jmclient/test/test_wallet_rpc.py

@@ -283,6 +283,42 @@ class TrialTestWRPC_DisplayWallet(WalletRPCTestBase, unittest.TestCase):
     def unauthorized_session_request_handler(self, response, code):
         assert code == 401
 
+    @defer.inlineCallbacks
+    def test_recover_wallet(self):
+        # before starting, we have to shut down the existing
+        # wallet service (usually this would be `lock`):
+        self.daemon.services["wallet"] = None
+        self.daemon.stopService()
+        self.daemon.auth_disabled = False
+
+        wfn1 = self.get_wallet_file_name(1)
+        self.wfnames = [wfn1]
+        agent = get_nontor_agent()
+        root = self.get_route_root()
+
+        addr = root + "/wallet/recover"
+        addr = addr.encode()
+        body = BytesProducer(json.dumps({"walletname": wfn1,
+                "password": "hunter2", "wallettype": "sw-fb",
+                "seedphrase": "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about"}).encode())
+        # Note: the recover wallet response is identical to
+        # the create wallet response
+        yield self.do_request(agent, b"POST", addr, body,
+                              self.process_create_wallet_response)
+
+        # Sanity check of startup; does a auth-ed session request succeed?
+        yield self.do_session_request(agent, root,
+            self.authorized_session_request_handler, token=self.jwt_token)
+        # What about display?
+        addr = self.get_route_root()
+        addr += "/wallet/"
+        addr += self.daemon.wallet_name
+        addr += "/display"
+        addr = addr.encode()
+        self.daemon.auth_disabled = True
+        yield self.do_request(agent, b"GET", addr, None,
+                              self.process_empty_wallet_display_response)
+
     @defer.inlineCallbacks
     def test_create_list_lock_unlock(self):
         """ A batch of tests in sequence here,
@@ -422,6 +458,12 @@ class TrialTestWRPC_DisplayWallet(WalletRPCTestBase, unittest.TestCase):
         yield self.do_request(agent, b"GET", addr, None,
                               self.process_wallet_display_response)
 
+    def process_empty_wallet_display_response(self, response, code):
+        assert code == 200
+        json_body = json.loads(response.decode("utf-8"))
+        wi = json_body["walletinfo"]
+        assert float(wi["total_balance"]) == 0.0 #?
+
     def process_wallet_display_response(self, response, code):
         assert code == 200
         json_body = json.loads(response.decode("utf-8"))