diff --git a/.gitignore b/.gitignore index 9a13be3..344f7c0 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,7 @@ scripts/commitmentlist tmp/ wallets/ scripts/jm-tx-history.txt +scripts/snicker/joinmarket.cfg +scripts/snicker/snicker-proposals.txt +scripts/snicker/candidates.txt + diff --git a/docs/SNICKER.md b/docs/SNICKER.md new file mode 100644 index 0000000..1ec28ae --- /dev/null +++ b/docs/SNICKER.md @@ -0,0 +1,349 @@ +# HOW TO USE THE SNICKER FEATURES IN JOINMARKET + +# Contents + +1. [Basic concepts and definition](#basic) + + a. [Quick read: advice](#quick) + + b. [Slightly longer description](#longer) + + c. [Proof of work for anti-spam](#pow) + + d. [Servers](#servers) + + e. [New script tools](#scripts) + +2. [Updating config](#configure) + +3. [Alternative to mainnet usage](#network) + +4. [Running SNICKER as a receiver](#receiver) + + a. [Manually running the receiver](#manually) + + b. [Running a yield-generator with SNICKER active](#yieldgen) + +5. [Use of wallets](#wallets) + +6. [Checking for SNICKER coinjoins](#checking) + +7. [A testing workflow](#testing) + +8. [Appendix: Example SNICKER transaction](#appendix1) + + + +## Basic concepts and definition + + + +### Quick read: advice + +For the time constraints, consider these points: + +* This is *somewhat* experimental; better run it on signet for now, mainnet is not advised. +* If you do run it on mainnet, make an effort to keep backups of your jmdat wallet file; recovery with seed only is possible (a tool is provided), but it's a pain. +* This basically allows coinjoins to be proposed and executed without any interaction by the participants, even over a message channel. You can run it passsively in a yield generator, for example. You can even be paid some small amount of sats for that to happen. But the coinjoins are only 2-party. + + + +### Slightly longer read on what this is: + +For formal specs as currently implemented, please use [this](https://gist.github.com/AdamISZ/2c13fb5819bd469ca318156e2cf25d79). Less technical description [here](https://joinmarket.me/blog/blog/snicker/). + +Essentially, this is a two party but non-interactive protocol. The **proposer** will identify, on the blockchain, a candidate transaction where he has some confidence that one or more inputs are owned by the same party as one output, and that that party has SNICKER receiver functionality. +Given those conditions, he'll create one or more **proposals** which are of form `base64-encoded-ECIES-encrypted-PSBT,hex-encoded-pubkey` (the separator is literally a comma; this is ASCII encoded), and send them to a **snicker-server** which is hosted at an onion address (possibly TLS but let's stick with onion for now, it's easier). They could also be sent manually. We'll talk more about these two possibilities below. + +To understand the step-by-step of how this is done "under the hood", you may find the [section on testing](#testing) a useful read. If you're only interested in "switching on this feature", notice this is not advised on mainnet (see the section on [alternatives to mainnet](#network)), but read more below about editing the config and switching it on in a yieldgenerator. + + + +### Proof of work for mild anti-spam + +As implemented here, in fact, the proposer attaches a proof of work in the form of a 10-byte nonce appended to the end of the above string (hex encoded; so in fact: `base64-encoded-ECIES-encrypted-PSBT,hex-encoded-pubkey,hex-encoded-nonce` is what is sent over the wire). This nonce is grinded to get a 32-byte-truncated-hash512 of that string to be less than a target calculated by a request number of bits from the server. The target bits is requested by the proposer with a `GET /target` request to the server, before sending the proposals themselves with a `POST /` request with the proposals in the body. For the proof of work, see `jmbase/jmbase/proof_of_work.py`; it's pretty elementary. Note of course this is no defence against a serious attempt to jam the system, it's only a "script-kiddie-defence", so to speak. + + + +### Servers + +The **snicker-server** just hosts the proposals and lets others read them. For the purpose of testing it's fine that we don't have a very sophisticated version of this, but for now note: +* You can run tests using the `-n` option of `create-snicker-proposal.py` to just output to terminal instead of uploading to server; this may be in particular more useful for local tests where it's only your own wallets involved. +* The server serves only over a Tor hidden service. +* It stores the accepted proposals in a sqlite3 database `proposals.db`. The table is `proposals` and has only two fields: `pubkey` and `proposal` (see above). +* Defends against spam with proof of work as per above (this is very limited but better than nothing). +* Currently has NO maintenance or performance feature such as flushing out proposals after a time limit, or allowing filtered queries. This seems the most obvious way to improve what exists, here. + + + +### New script tools + +The new tools for SNICKER are in the directory `scripts/snicker`, consisting of: +* `snicker-seed-tx.py` - create a fake SNICKER transaction in your own wallet. +* `snicker-finder.py` - scan recent blocks for Joinmarket or SNICKER candidate transactions. +* `create-snicker-proposal.py` - takes transactions found from the above and makes proposals, uploading them to a server or outputting to command line. +* `snicker-server.py` - implements a simple server serving over *.onion, with a sqlite database to store proposals, and defends against spam only mildly with a proof of work requirement (see below). +* `receive-snicker.py` - polls above server to read new proposals, parse them and broadcasts completed SNICKER coinjoins when found, storing the new keys as imports (see details on wallet handling below). +* `snicker-recovery.py` - can be used to recover a wallet from seedphrase which contains SNICKER utxos, though it needs (possibly multiple) rescanblockchain calls (and informs the user how to do this, including blockheights). + + + +## Updating config + +Recreate your joinmarket.cfg in the usual way. You can then edit the new `[SNICKER]` section as desired: + +``` +[SNICKER] + +# any other value than 'true' will be treated as False, +# and no SNICKER actions will be enabled in that case: +enabled = false + +# in satoshis, we require any SNICKER to pay us at least +# this much (can be negative), otherwise we will refuse +# to sign it: +lowest_net_gain = 0 + +# comma separated list of servers (if port is omitted as :port, it +# is assumed to be 80) which we will poll against (all, in sequence); note +# that they are allowed to be *.onion or cleartext servers, and no +# scheme (http(s) etc) needs to be added to the start. +servers = cn5lfwvrswicuxn3gjsxoved6l2gu5hdvwy5l3ev7kg6j7lbji2k7hqd.onion, + +# how many minutes between each polling event to each server above: +polling_interval_minutes = 60 +``` + +Notice that it is of course *NOT* enabled by default, so switch that to `true`. + +If you are running tests, a 60 minute polling interval is slow, feel free to cut it down to a minute or two. + +The default server is currently running on a VPS. As mentioned above, you can easily run your own server with `snicker-server.py`. It is possible to poll multiple servers, comma separated in the list. + + + +## Alternative to mainnet usage + +Choosing the network for this function means editing the `[BLOCKCHAIN]` section in `joinmarket.cfg`, just as for other Joinmarket functionality. + +**Regtest** is a good option if you are interested in testing functionality quickly, on your own. See [here](TESTING.md) for some info on regtest setup. + +**Signet** (a new testnet) will be helpful especially if you intend to test with others. This can be done by simply sharing proposals as per the `-n` comments above, or by sharing proposal server locations as onion addresses, and possibly communicating with other testers to identify candidate transactions (obviously this strays far from the intended way SNICKER will be used, but it is convenient to test workflow). +For more information about using Joinmarket with signet, see the [0.8.1 release notes](release-notes/release-notes-0.8.1.md) and [this gist](https://gist.github.com/AdamISZ/325716a66c7be7dd3fc4acdfce449fb1). + +**Testnet3** - it may be possible but it will be far less convenient than signet. + +**Mainnet** - as of Feb 2021, this isn't recommended yet; it should in theory work no different, but any usage would be at your own risk. + + + +## Running SNICKER as a receiver + +The **receiver** (unless handling manually with `-n` as per above) polls this server (for testing, you can make the polling loop fast; in real usage it should be slow), reads all the existing proposals using a `GET /` request with no parameters, and if it can decrypt and sanity check the transaction OK, it co-signs it and broadcasts it. Note: *the receiver wallet will store its new coins output from the coinjoin, as imported keys; they are not part of the HD tree, although derivable from history*. See `use of wallets` below for important notes on this aspect. + + + +### Manually running the receiver + +For this you can use the `receive-snicker.py` script as detailed above, passing the chosen wallet file as argument, and it will "one-shot" poll for proposals and process them, or, you can pass the base64 proposal manually instead. + + + +### Running SNICKER in a yield generator. + +This will presumably be the most normal way to be a SNICKER receiver over time; if `enabled=true` for SNICKER is in the above config settings, this will happen automatically, under the hood. Make sure the polling loop interval is not too fast if you leave this running longer term (even if a test bot). If valid proposals are found that follow our requirements, the transactions are broadcast. + + + +## Use of wallets + +**Wallet type** - please stick with native segwit (`native=true` in config *before* you generate), but you can also choose p2sh-p2wpkh, it should work. No other script type (including p2pkh) will work here. We don't want mixed script type SNICKER coinjoins. + +**Persistence in the wallet** - this is very important and not at all obvious! But, on regtest and testnet *by default*, we use hex seeds instead of wallet files and `VolatileStorage` (wallet storage in memory; wiped on shutdown). This is fine and convenient for many tests, but will not work for a key part of SNICKER - imported keys. + +The upshot - **make sure you actually generate wallet files for all wallets you're going to test SNICKER with**, otherwise you will not even see the created coins on the receiver side. + +Additionally, when you view the wallet with wallet-tool, you need to use `--recoversync`, as the default fast sync won't see imported keys. If you took these two steps, your tests should correctly show the post-SNICKER created coins. + + + +## Checking for SNICKER coinjoins + +SNICKER is logged to: + +`~/.joinmarket/logs/SNICKER/SNICKER-joinmarket-wallet-xxxxxx.log` + +i.e. there is one SNICKER log file for each wallet. +This should show all transactions that were detected and broadcast. + + + +## A testing workflow + +This is a scenario for a single user, using either regtest or signet. + +Generate a minimum of two joinmarket wallets with `python wallet-tool.py generate`, as noted above, native (or at least, both the same type). + +Fund them both. The receiver needs at least two utxos to create the seed transaction. + +Create a seed fake-SNICKER transaction in the receiver wallet, using the script `snicker-seed-tx.py`. + +Start the test server. Navigate to `scripts/snicker` and run `python snicker-server.py` - no arguments should be needed, and this will generate an onion running serving on port 80; the onion hostname is displayed: + +``` +(jmvenv) waxwing@here~/testjminstall/joinmarket-clientserver/scripts/snicker$ python snicker-server.py +User data location: +Attempting to start onion service on port: 80 ... +Your hidden service is available: +xpkqk2cy2h2ay5iecwcod5ka36nxj2tsiyczk2w5c6o7h5g57w3xg4id.onion +``` + +This is ephemeral, obviously we intend the real servers to be long-running. The one in the default config should exist and be long-running already, but of course your tests don't need to rely on this. Add: + +``` + +[SNICKER] +enabled = true +servers= xpkqk2cy2h2ay5iecwcod5ka36nxj2tsiyczk2w5c6o7h5g57w3xg4id.onion, +``` +... to a `joinmarket.cfg` that you add inside `scripts/snicker`, by copying it from `scripts/` or wherever you keep your testing `joinmarket.cfg` file. (This manual annoyance is part of testing, it won't be needed in mainnet usage of course ... alternatively just put a signet/regtest custom `joinmarket.cfg` in your `.joinmarket` folder, but this is hardly less annoying). + +`servers=` requires a comma separated list. + +You're now ready to do the two steps: (a) create a proposal and upload it, (b) download proposals (as the receiver identity/wallet) and complete coinjoins. It could be different people doing (a) and (b) of course but here we're assuming one tester doing everything (see two wallets above). + +### Creating one or more proposals + +Having done the above seed transaction, do a scan operation to find the candidate: + +`cd script/snicker; python snicker-finder.py --datadir=. 330` + +Here, 330 is the starting block number on my regtest blockchain; the ending block is the current block. On signet use a block explorer to find current height (or `bitcoin-cli getblockchaininfo`). It will look for all transactions with a SNICKER pattern and you should see returned something like this: + +``` +2020-12-28 16:09:04,329 [INFO] Finished processing block: 790 +2020-12-28 16:09:04,334 [INFO] Found SNICKER transaction: 32f80807b3ba4ca477b25e8ab608a8a3134a34c8c3787cad95c653d1805d7533 in block: 791 +2020-12-28 16:09:04,338 [INFO] Finished processing block: 791 +2020-12-28 16:09:04,340 [INFO] Finished processing block: 792 +done +``` + +Then look in `./candidates.txt` to find the details of the identified transaction, including its full hex, which you need to copy: + +``` +2020-12-16 11:13:01,708 [INFO] { + "hex": "0200000000010138e8a90b3df7740b9d5f5ae9af2cf6769f314d290b2e12bf25bfa4aae2c0cbe20000000000feffffff0280ba8c010000000016001471b09afbac6204627225c10f3a8d4a0749364fdb6d7c36220000000016001447ae59f32c504cbb56e18b77f7842fb58b55025b02473044022075354351ad4c619ba662f9abd25e8ee434f8381795001606a29fa959d36aeb7f022018f8bf1ec0407dad586baeb7e4d977aaacbd8fb15579293e6d739ad69ac3c6cf012103f8e827464fb83209c194376c53ae8f4e7ab5f1baf0948705fec6dd421f2b65c37a020000", + "inputs": [ + ................. +Full transaction hex for creating a proposal is found in the above. +The unspent indices are: 0 1 2 +``` + +Copy that fully signed transaction hex, and note the unspent outputs. You're going to assume (in this case correctly of course) that all of the inputs are valid options for the SNICKER public key (why? because it was a *seed* transaction, it was a fake SNICKER, and so all the inputs belonged to the same wallet). + +At this point you're ready to run the proposal creator: + +``` +python create-encrypted-proposal.py --datadir=. proposerwallet.jmdat "0200000000010138e8a90b3df7740b9d5f5ae9af2cf6769f314d290b2e12bf25bfa4aae2c0cbe20000000000feffffff0280ba8c010000000016001471b09afbac6204627225c10f3a8d4a0749364fdb6d7c36220000000016001447ae59f32c504cbb56e18b77f7842fb58b55025b02473044022075354351ad4c619ba662f9abd25e8ee434f8381795001606a29fa959d36aeb7f022018f8bf1ec0407dad586baeb7e4d977aaacbd8fb15579293e6d739ad69ac3c6cf012103f8e827464fb83209c194376c53ae8f4e7ab5f1baf0948705fec6dd421f2b65c37a020000" 0 1 100 -m1 -n +``` +Obviously see the `--help` for details, but in this example we chose: + +* input index 0 to source the pubkey for the encryption. +* output index 1 for the coin we want the receiver to spend with us in the coinjoin. +* 100 sats as the amount we will bump their output by as an incentive to do the coinjoin (you *can* make this number negative, to receive, instead - the proposer is paying the tx fee otherwise, note). +* mixdepth 1 as the mixdepth from which we source *our* coins for the coinjoin. Make sure of course that mixdepth 1 has at least a little bit more bitcoin than the size of that output at index 1 aforementioned. +* The `-n` option can be used to output the proposal to stdout (it is base64+hex so copy-pasteable). + +If you choose not to use -n, but instead use a proposals server as above, then assuming it connects to the server OK, you will see: + +``` +Response from server: http://xpkqk2cy2h2ay5iecwcod5ka36nxj2tsiyczk2w5c6o7h5g57w3xg4id.onion was: 1 proposals-accepted +``` + +### Receiving the created proposals. + +The last phase is pretty simple, if it works - just run the receiver script (from `scripts/snicker`) as follows: + +``` +python receive-snicker.py --datadir=. receiver.jmdat [proposal] +User data location: . +2020-12-16 11:43:03,779 [DEBUG] rpc: getblockchaininfo [] +2020-12-16 11:43:03,781 [DEBUG] rpc: getnewaddress [] +Enter passphrase to decrypt wallet: +2020-12-16 11:43:07,501 [DEBUG] rpc: listaddressgroupings [] +2020-12-16 11:43:07,562 [DEBUG] Fast sync in progress. Got this many used addresses: 3 +2020-12-16 11:43:08,075 [DEBUG] rpc: listunspent [0] +2020-12-16 11:43:08,216 [DEBUG] bitcoind sync_unspent took 0.14214825630187988sec +2020-12-16 11:43:08,280 [WARNING] Cannot listen on port 27183, trying next port +2020-12-16 11:43:08,281 [WARNING] Cannot listen on port 27184, trying next port +2020-12-16 11:43:08,281 [WARNING] Cannot listen on port 27185, trying next port +2020-12-16 11:43:08,281 [INFO] Listening on port 27186 +2020-12-16 11:43:08,282 [INFO] (SNICKER) Listening on port 26186 +2020-12-16 11:43:08,282 [INFO] Starting transaction monitor in walletservice +2020-12-16 11:43:08,339 [INFO] Starting SNICKER polling loop +2020-12-16 11:43:22,676 [DEBUG] rpc: sendrawtransaction ['020000000001028ffa6a6f0184ed8123993273ecbb4af82d1b1c0963c815fec4e92525eaba56b30000000000ffffffffa5015509e0e241ef25ee7ccc1936295c908e572cb222105e16c197d66f0599640000000000ffffffff03e4ba8c0100000000160014190ec76b7843f47bc367b65119b98c32074536255dfd5e0a00000000160014d38fa4a6ac8db7495e5e2b5d219dccd412dd9baee4ba8c01000000001600147b4676f859b993257bc8d5880650fcab470db8a1024830450221008480d553177a020f58ca0e45b9e20aa027305a279a3de1014f55ff22909b89b1022054e848285ee60c169b5de19bb4d3637b606ff14bc4cca4506ad05a42fff6af400121029a82a00f05d023f188dfd1db82ef8ec136b0500bbd33bb1f65930c5b74e3199802463043021f01d3f4567c32fc0c5c0cd33db233a3c74100a36940d743b72042b55e60b89d022073ab203ad0fee389f2a2c9e62197244cea95b07ae78a5516ca9f866a8e348d2c01210245d8623c4b06505dffd21bdd314a84b73afe2b9d49a93fe89397b48a85b718bd00000000'] +2020-12-16 11:43:22,678 [INFO] Successfully broadcast SNICKER coinjoin: 33ec857df09030140391529295412434cced8191626024f937426b7859a21947 +2020-12-16 11:43:23,359 [INFO] Removed utxos= +b356baea2525e9c4fe15c863091c1b2df84abbec7332992381ed84016f6afa8f:0 - path: m/84'/1'/4'/0/0, address: bcrt1qwxcf47avvgzxyu39cy8n4r22qaynvn7mc359ap, value: 26000000 +2020-12-16 11:43:23,360 [INFO] Added utxos= +33ec857df09030140391529295412434cced8191626024f937426b7859a21947:0 - path: imported/1/0, address: bcrt1qry8vw6mcg068hsm8keg3nwvvxgr52d3923gg45, value: 26000100 +``` + +Obviously this is the ideal case: if no errors occur. If invalid proposals, or proposals on coins that no longer exist because you already spent them, are encountered, logging messages are displayed to that effect. + +If you did choose the `-n` option then you can pass the copy-pasted proposal on the command line and it will just process that instead of polling. + +### Other kinds of testing + +The above is the baseline workflow. Additionally, you can test: + +#### Wallet recovery from seed + +This case is already somewhat tricky in Joinmarket, but for the worst possible scenario of only having a seedphrase and no address imports in the wallet and no Joinmarket jmdat file, and having used SNICKER recently and not spent those coins, recovery is particularly tricky (which is why users who enable this feature must be warned that it's very important not to lose the jmdat file). However even in this case full recovery is possible, using the script `snicker-recovery.py`. To fully test this try making multiple SNICKER transactions in a wallet, then deleting the jmdat and creating a new Core wallet (on the same regtest instance of course!), enabling it in joinmarket.cfg, then running `wallet-tool.py recover` with the seedphrase, then running the aforementioned snicker recovery script; it will prompt you to `rescanblockchain` from certain heights, potentially more than once; the reason for this is that Core cannot find arbitrarily the transactions which spend custom keys which we discover during wallet recovery, we need to import and rescan before going to the next step, although this will only be an edge case. + + + +## Appendix: Example SNICKER transaction + +This is what is produced by `print(jmbitcoin.human_readable_transaction(jmbitcoin.CTransaction.deserialize(jmbase.hextobin('020000000001028ffa6a6f0184ed8123993273ecbb4af82d1b1c0963c815fec4e92525eaba56b30000000000ffffffffa5015509e0e241ef25ee7ccc1936295c908e572cb222105e16c197d66f0599640000000000ffffffff03e4ba8c0100000000160014190ec76b7843f47bc367b65119b98c32074536255dfd5e0a00000000160014d38fa4a6ac8db7495e5e2b5d219dccd412dd9baee4ba8c01000000001600147b4676f859b993257bc8d5880650fcab470db8a1024830450221008480d553177a020f58ca0e45b9e20aa027305a279a3de1014f55ff22909b89b1022054e848285ee60c169b5de19bb4d3637b606ff14bc4cca4506ad05a42fff6af400121029a82a00f05d023f188dfd1db82ef8ec136b0500bbd33bb1f65930c5b74e3199802463043021f01d3f4567c32fc0c5c0cd33db233a3c74100a36940d743b72042b55e60b89d022073ab203ad0fee389f2a2c9e62197244cea95b07ae78a5516ca9f866a8e348d2c01210245d8623c4b06505dffd21bdd314a84b73afe2b9d49a93fe89397b48a85b718bd00000000'))))`: + +``` +{ + "hex": "02000000000102578770b2732aed421ffe62d54fd695cf281ca336e4f686d2adbb2e8c3bedb2570000000000ffffffff4719a259786b4237f92460629181edcc3424419592529103143090f07d85ec330100000000ffffffff0324fd9b0100000000160014d38fa4a6ac8db7495e5e2b5d219dccd412dd9bae24fd9b0100000000160014564aead56de8f4d445fc5b74a61793b5c8a819667af6c208000000001600146ec55c2e1d1a7a868b5ec91822bf40bba842bac502473044022078f8106a5645cc4afeef36d4addec391a5b058cc51053b42c89fcedf92f4db1002200cdf1b66a922863fba8dc1b1b1a0dce043d952fa14dcbe86c427fda25e930a53012102f1f750bfb73dbe4c7faec2c9c301ad0e02176cd47bcc909ff0a117e95b2aad7b02483045022100b9a6c2295a1b0f7605381d416f6ed8da763bd7c20f2402dd36b62dd9dd07375002207d40eaff4fc6ee219a7498abfab6bdc54b7ce006ac4b978b64bff960fbf5f31e012103c2a7d6e44acdbd503c578ec7d1741a44864780be0186e555e853eee86e06f11f00000000", + "inputs": [ + { + "outpoint": "57b2ed3b8c2ebbadd286f6e436a31c28cf95d64fd562fe1f42ed2a73b2708757:0", + "scriptSig": "", + "nSequence": 4294967295, + "witness": "02473044022078f8106a5645cc4afeef36d4addec391a5b058cc51053b42c89fcedf92f4db1002200cdf1b66a922863fba8dc1b1b1a0dce043d952fa14dcbe86c427fda25e930a53012102f1f750bfb73dbe4c7faec2c9c301ad0e02176cd47bcc909ff0a117e95b2aad7b" + }, + { + "outpoint": "33ec857df09030140391529295412434cced8191626024f937426b7859a21947:1", + "scriptSig": "", + "nSequence": 4294967295, + "witness": "02483045022100b9a6c2295a1b0f7605381d416f6ed8da763bd7c20f2402dd36b62dd9dd07375002207d40eaff4fc6ee219a7498abfab6bdc54b7ce006ac4b978b64bff960fbf5f31e012103c2a7d6e44acdbd503c578ec7d1741a44864780be0186e555e853eee86e06f11f" + } + ], + "outputs": [ + { + "value_sats": 27000100, + "scriptPubKey": "0014d38fa4a6ac8db7495e5e2b5d219dccd412dd9bae", + "address": "bc1q6w86ff4v3km5jhj79dwjr8wv6sfdmxaw2dytjc" + }, + { + "value_sats": 27000100, + "scriptPubKey": "0014564aead56de8f4d445fc5b74a61793b5c8a81966", + "address": "bc1q2e9w44tdar6dg30utd62v9unkhy2sxtxtqrthh" + }, + { + "value_sats": 146994810, + "scriptPubKey": "00146ec55c2e1d1a7a868b5ec91822bf40bba842bac5", + "address": "bc1qdmz4ctsarfagdz67eyvz906qhw5y9wk9dqpuea" + } + ], + "txid": "ca606efc5ba8f6669ba15e9262e5d38e745345ea96106d5a919688d1ff0da0cc", + "nLockTime": 0, + "nVersion": 2 +} +``` + diff --git a/jmbase/jmbase/__init__.py b/jmbase/jmbase/__init__.py index 961a0a4..0a7d926 100644 --- a/jmbase/jmbase/__init__.py +++ b/jmbase/jmbase/__init__.py @@ -7,8 +7,9 @@ from .support import (get_log, chunks, debug_silence, jmprint, utxo_to_utxostr, EXIT_ARGERROR, EXIT_FAILURE, EXIT_SUCCESS, hexbin, dictchanger, listchanger, JM_WALLET_NAME_PREFIX, JM_APP_NAME, - IndentedHelpFormatterWithNL) -from .twisted_utils import stop_reactor + IndentedHelpFormatterWithNL, wrapped_urlparse) +from .proof_of_work import get_pow, verify_pow +from .twisted_utils import stop_reactor, is_hs_uri, get_tor_agent, get_nontor_agent from .bytesprod import BytesProducer from .commands import * diff --git a/jmbase/jmbase/commands.py b/jmbase/jmbase/commands.py index d0db13f..7144ff3 100644 --- a/jmbase/jmbase/commands.py +++ b/jmbase/jmbase/commands.py @@ -228,3 +228,56 @@ class JMTXBroadcast(JMCommand): broadcast. """ arguments = [(b'txhex', Unicode())] + +"""SNICKER related commands. +""" + +class SNICKERReceiverInit(JMCommand): + """ Initialization data for a SNICKER service. + See documentation of `netconfig` in + jmdaemon.HTTPPassThrough.on_INIT + """ + arguments = [(b'netconfig', Unicode())] + +class SNICKERProposerInit(JMCommand): + """ As for receiver. + """ + arguments = [(b'netconfig', Unicode())] + +class SNICKERReceiverUp(JMCommand): + arguments = [] + +class SNICKERProposerUp(JMCommand): + arguments = [] + +class SNICKERReceiverGetProposals(JMCommand): + arguments = [] + +class SNICKERReceiverProposals(JMCommand): + """ Sends the retrieved proposal list from + a specific server, from daemon back to client. + """ + arguments = [(b'proposals', BigUnicode()), + (b'server', Unicode())] + +class SNICKERProposerPostProposals(JMCommand): + """ Sends a list of proposals to be uploaded + to a server. + """ + arguments = [(b'proposals', BigUnicode()), + (b'server', Unicode())] + +class SNICKERProposalsServerResponse(JMCommand): + arguments = [(b'response', Unicode()), + (b'server', Unicode())] + +class SNICKERServerError(JMCommand): + arguments = [(b'server', Unicode()), + (b'errorcode', Integer())] + +class SNICKERRequestPowTarget(JMCommand): + arguments = [(b'server', Unicode())] + +class SNICKERReceivePowTarget(JMCommand): + arguments = [(b'server', Unicode()), + (b'targetbits', Integer())] diff --git a/jmbase/jmbase/proof_of_work.py b/jmbase/jmbase/proof_of_work.py new file mode 100644 index 0000000..fa16386 --- /dev/null +++ b/jmbase/jmbase/proof_of_work.py @@ -0,0 +1,41 @@ +import hashlib +from .support import bintohex + +def get_pow(data, noncelen=10, hashfn=hashlib.sha512, + hashlen=64, nbits=1, truncate=0, maxiterations=10**8): + """ Arguments: + data - a string of bytes. + noncelen - an int, the number of additional bytes to be appended + to the bytestring `data` which will be used for grinding. + hashfn - a function that outputs a finalized hash state that can + be converted to a bytestring with .digest() (see hashlib). + hashlen - the length of the bytestring created with the .digest() + call just mentioned. + nbits - an integer, the number of bits of proof of work required. + truncate - an integer number of bytes to be truncated from the end + of the hash digest created. + maxiterations - an integer, how many grinding attempts maximum allowed + to attempt to reach the target, before giving up. + Returns: + (nonceval, pow-preimage, niterations) + where pow-preimage is data+nonce-in-bytes + or + (None, failure-reason, None) + """ + maxbits = (hashlen-truncate)*8 + pow_target = 2 ** (maxbits - nbits) + # note since we are using a trivial counter, two + # elements of returned tuple are the same, this needn't be the case. + for nonceval in range(maxiterations): + x = data + bintohex(nonceval.to_bytes(noncelen, "big")).encode( + "utf-8") + pow_candidate = hashfn(x).digest()[:truncate] + if int.from_bytes(pow_candidate, "big") < pow_target: + return (nonceval, x, nonceval) + return (None, "exceeded max-iterations: {}".format(maxiterations), None) + +def verify_pow(data, hashfn=hashlib.sha512, hashlen=64, nbits=1, truncate=0): + return int.from_bytes(hashfn(data).digest()[:truncate], + "big") < 2 ** ((hashlen - truncate) * 8 - nbits) + + \ No newline at end of file diff --git a/jmbase/jmbase/support.py b/jmbase/jmbase/support.py index 98e2e44..6ab7f80 100644 --- a/jmbase/jmbase/support.py +++ b/jmbase/jmbase/support.py @@ -5,6 +5,7 @@ from getpass import getpass from os import path, environ from functools import wraps from optparse import IndentedHelpFormatter +import urllib.parse as urlparse # JoinMarket version JM_CORE_VERSION = '0.8.2dev' @@ -287,4 +288,18 @@ def hexbin(func): newargs.append(_convert(arg)) return func(inst, *newargs, **kwargs) - return func_wrapper \ No newline at end of file + return func_wrapper + +def wrapped_urlparse(url): + """ This wrapper is unfortunately necessary as there appears + to be a bug in the urlparse handling of *.onion strings: + If http:// is prepended, the url parses correctly, but if it + is not, the .hostname property is erroneously None. + """ + if isinstance(url, str): + a, b = (".onion", "http://") + else: + a, b = (b".onion", b"http://") + if url.endswith(a) and not url.startswith(b): + url = b + url + return urlparse.urlparse(url) \ No newline at end of file diff --git a/jmbase/jmbase/twisted_utils.py b/jmbase/jmbase/twisted_utils.py index a5df2e2..67e974c 100644 --- a/jmbase/jmbase/twisted_utils.py +++ b/jmbase/jmbase/twisted_utils.py @@ -1,6 +1,43 @@ -from twisted.internet.error import ReactorNotRunning, AlreadyCancelled +from zope.interface import implementer +from twisted.internet.error import ReactorNotRunning from twisted.internet import reactor +from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.web.client import Agent, BrowserLikePolicyForHTTPS +from txtorcon.web import tor_agent +from twisted.web.iweb import IPolicyForHTTPS +from twisted.internet.ssl import CertificateOptions +from .support import wrapped_urlparse + +# txtorcon outputs erroneous warnings about hiddenservice directory strings, +# annoyingly, so we suppress it here: +import warnings +warnings.filterwarnings("ignore") + +""" This whitelister allows us to accept any cert for a specific + domain, and is to be used for testing only; the default Agent + behaviour of twisted.web.client.Agent for https URIs is + the correct one in production (i.e. uses local trust store). +""" +@implementer(IPolicyForHTTPS) +class WhitelistContextFactory(object): + def __init__(self, good_domains=None): + """ + :param good_domains: List of domains. The URLs must be in bytes + """ + if not good_domains: + self.good_domains = [] + else: + self.good_domains = good_domains + # by default, handle requests like a browser would + self.default_policy = BrowserLikePolicyForHTTPS() + + def creatorForNetloc(self, hostname, port): + # check if the hostname is in the the whitelist, + # otherwise return the default policy + if hostname in self.good_domains: + return CertificateOptions(verify=False) + return self.default_policy.creatorForNetloc(hostname, port) def stop_reactor(): """ The value of the bool `reactor.running` @@ -14,3 +51,26 @@ def stop_reactor(): reactor.stop() except ReactorNotRunning: pass + + + +def is_hs_uri(s): + x = wrapped_urlparse(s) + if x.hostname.endswith(".onion"): + return (x.scheme, x.hostname, x.port) + return False + +def get_tor_agent(socks5_host, socks5_port): + torEndpoint = TCP4ClientEndpoint(reactor, socks5_host, socks5_port) + return tor_agent(reactor, torEndpoint) + +def get_nontor_agent(tls_whitelist=[]): + """ The tls_whitelist argument must be a list of hosts for which + TLS certificate verification may be omitted, default none. + """ + if len(tls_whitelist) == 0: + agent = Agent(reactor) + else: + agent = Agent(reactor, + contextFactory=WhitelistContextFactory(tls_whitelist)) + return agent \ No newline at end of file diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index 2ca1e13..8c5615d 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -32,4 +32,5 @@ from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL, SIGVERSION_WITNESS_V0, CScriptWitness) from bitcointx.core.psbt import (PartiallySignedTransaction, PSBT_Input, PSBT_Output) +from .blocks import get_transactions_in_block diff --git a/jmbitcoin/jmbitcoin/blocks.py b/jmbitcoin/jmbitcoin/blocks.py new file mode 100644 index 0000000..95eb0da --- /dev/null +++ b/jmbitcoin/jmbitcoin/blocks.py @@ -0,0 +1,65 @@ +""" +Module implementing actions that can be taken on +network-serialized bitcoin blocks. +""" + +import struct +from jmbase import hextobin, bintohex +from bitcointx.core import CBitcoinTransaction +from bitcointx.core.serialize import VarIntSerializer, VarBytesSerializer + +def decode_varint(data): + n, tail = VarIntSerializer.deserialize_partial(data) + head = data[0: len(data) - len(tail)] + return n, len(head) + +def get_transactions_in_block(block): + """ `block` is hex output from RPC `getblock`. + Return: + yields the block's transactions, type CBitcoinTransaction + """ + block = hextobin(block) + + # Skipping the header + transaction_data = block[80:] + + # Decoding the number of transactions, offset is the size of + # the varint (1 to 9 bytes) + n_transactions, offset = decode_varint(transaction_data) + + for i in range(n_transactions): + # This 'strat' of reading in small chunks optimistically is taken from: + # https://github.com/alecalve/python-bitcoin-blockchain-parser/blob/7a9e15c236b10d2a6dff5e696801c0641af72628/blockchain_parser/utils.py + # Try from 1024 (1KiB) -> 1073741824 (1GiB) slice widths + for j in range(0, 20): + try: + offset_e = offset + (1024 * 2 ** j) + transaction = CBitcoinTransaction.deserialize( + transaction_data[offset:offset_e], allow_padding=True) + yield transaction + break + except: + continue + + # Skipping to the next transaction + offset += len(transaction.serialize()) + +""" Example block from a regtest: + +# Found using `getblockhash 222` followed by `getblock 0`: + +0000003066327ecf2f3e72ec43f358c9c7b34f47374f23f4fcce965d4e18273a5b98f325d11b3b9c3a592c830d49f6281d4055f5732a79a19f9bd8d4afad729772cbf393fa7bdf5fffff7f2000000000010200000 +00001010000000000000000000000000000000000000000000000000000000000000000ffffffff0502de000101ffffffff0200f902950000000017a914d2e1d0ea5135f0cbeb4aef06e3cee785d394876a870000000000000000266a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf90120000000000000000000000000000000000000000000000000000000000000000000000000 + +Gives (x = above block): + +y = list(btc.get_block_transactions(x)) +>>> y[0] +CBitcoinTransaction([CBitcoinTxIn(CBitcoinOutPoint(), CBitcoinScript([x('de00'), x('01')]), 0xffffffff)], +[CBitcoinTxOut(25.0*COIN, CBitcoinScript([OP_HASH160, x('d2e1d0ea5135f0cbeb4aef06e3cee785d394876a'), OP_EQUAL])), +CBitcoinTxOut(0.0*COIN, CBitcoinScript([OP_RETURN, x('aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf9')]))], +0, 2, CBitcoinTxWitness([CBitcoinTxInWitness(CScriptWitness([x('0000000000000000000000000000000000000000000000000000000000000000')]))])) + +(coinbase transaction) +""" + diff --git a/jmbitcoin/jmbitcoin/secp256k1_ecies.py b/jmbitcoin/jmbitcoin/secp256k1_ecies.py index 8e82108..924bcbd 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_ecies.py +++ b/jmbitcoin/jmbitcoin/secp256k1_ecies.py @@ -34,14 +34,11 @@ def aes_decrypt(key, data, iv): return dec_data def ecies_encrypt(message, pubkey): - """ Take a privkey in raw byte serialization, - and a pubkey serialized in compressed, binary format (33 bytes), - and output the shared secret as a 32 byte hash digest output. - The exact calculation is: - shared_secret = SHA256(privkey * pubkey) - .. where * is elliptic curve scalar multiplication. - See https://github.com/bitcoin/bitcoin/blob/master/src/secp256k1/src/modules/ecdh/main_impl.h - for implementation details. + """ Take a message in bytes and a secp256k1 public key + in compressed byte serialization, and output the + ECIES encryption, using magic bytes as defined in this module, + sha512 for the key expansion, and AES-CBC for the encryption; + these choices are aligned with that used by Electrum. """ # create an ephemeral pubkey for this encryption: while True: diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index a59e3d3..beaa80d 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -3,13 +3,14 @@ # note, only used for non-cryptographic randomness: import random import json +import itertools # needed for single sha256 evaluation, which is used # in bitcoin (p2wsh) but not exposed in python-bitcointx: import hashlib from jmbitcoin.secp256k1_main import * from jmbase import bintohex, utxo_to_utxostr -from bitcointx.core import (CMutableTransaction, Hash160, CTxInWitness, +from bitcointx.core import (CMutableTransaction, CTxInWitness, CMutableOutPoint, CMutableTxIn, CTransaction, CMutableTxOut, CTxIn, CTxOut, ValidationError) from bitcointx.core.script import * @@ -314,7 +315,7 @@ def mktx(ins, outs, version=1, locktime=0): def make_shuffled_tx(ins, outs, version=1, locktime=0): """ Simple wrapper to ensure transaction inputs and outputs are randomly ordered. - Can possibly be replaced by BIP69 in future + NB: This mutates ordering of `ins` and `outs`. """ random.shuffle(ins) random.shuffle(outs) @@ -332,3 +333,103 @@ def verify_tx_input(tx, i, scriptSig, scriptPubKey, amount=None, witness=None): except ValidationError as e: return False return True + +def extract_witness(tx, i): + """Given `tx` of type CTransaction, extract, + as a list of objects of type CScript, which constitute the + witness at the index i, followed by "success". + If the witness is not present for this index, (None, "errmsg") + is returned. + Callers must distinguish the case 'tx is unsigned' from the + case 'input is not type segwit' externally. + """ + assert isinstance(tx, CTransaction) + assert i >= 0 + if not tx.has_witness(): + return None, "Tx witness not present" + if len(tx.vin) < i: + return None, "invalid input index" + witness = tx.wit.vtxinwit[i] + return (witness, "success") + +def extract_pubkey_from_witness(tx, i): + """ Extract the pubkey used to sign at index i, + in CTransaction tx, assuming it is of type p2wpkh + (including wrapped segwit version). + Returns (pubkey, "success") or (None, "errmsg"). + """ + witness, msg = extract_witness(tx, i) + sWitness = [a for a in iter(witness.scriptWitness)] + if not sWitness: + return None, msg + else: + if len(sWitness) != 2: + return None, "invalid witness for p2wpkh." + if not is_valid_pubkey(sWitness[1], True): + return None, "invalid pubkey in witness" + return sWitness[1], "success" + +def get_equal_outs(tx): + """ If 2 or more transaction outputs have the same + bitcoin value, return then as a list of CTxOuts. + If there is not exactly one equal output size, return False. + """ + retval = [] + l = [x.nValue for x in tx.vout] + eos = [i for i in l if l.count(i)>=2] + if len(eos) > 0: + eos = set(eos) + if len(eos) > 1: + return False + for i, vout in enumerate(tx.vout): + if vout.nValue == list(eos)[0]: + retval.append((i, vout)) + assert len(retval) > 1 + return retval + +def is_jm_tx(tx, min_cj_amount=75000, min_participants=3): + """ Identify Joinmarket-patterned transactions. + TODO: this should be in another module. + Given a CBitcoinTransaction tx, check: + nins >= number of coinjoin outs (equal sized) + non-equal outs = coinjoin outs or coinjoin outs -1 + at least 3 coinjoin outs (2 technically possible but excluded) + also possible to try to get clever about fees, but won't bother. + note: BlockSci's algo additionally addresses subset sum, so will + give better quality data, but this is kept simple for now. + We filter out joins with less than 3 participants as they are + not really in Joinmarket "correct usage" and there will be a lot + of false positives. + We filter out "joins" less than 75000 sats as they are unlikely to + be Joinmarket and there tend to be many low-value false positives. + Returns: + (False, None) for non-matches + (coinjoin amount, number of participants) for matches. + """ + def assumed_cj_out_num(nout): + """Return the value ceil(nout/2) + """ + x = nout//2 + if nout %2: return x+1 + return x + + def most_common_value(x): + return max(set(x), key=x.count) + + assumed_coinjoin_outs = assumed_cj_out_num(len(tx.vout)) + if assumed_coinjoin_outs < min_participants: + return (False, None) + if len(tx.vin) < assumed_coinjoin_outs: + return (False, None) + outvals = [x.nValue for x in tx.vout] + # it's not possible for the coinjoin out to not be + # the most common value: + mcov = most_common_value(outvals) + if mcov < min_cj_amount: + return (False, None) + cjoutvals = [x for x in outvals if x == mcov] + if len(cjoutvals) != assumed_coinjoin_outs: + return (False, None) + # number of participants is the number of assumed + # coinjoin outputs: + return (mcov, assumed_coinjoin_outs) diff --git a/jmbitcoin/jmbitcoin/snicker.py b/jmbitcoin/jmbitcoin/snicker.py index 05d5b4b..1970a51 100644 --- a/jmbitcoin/jmbitcoin/snicker.py +++ b/jmbitcoin/jmbitcoin/snicker.py @@ -6,6 +6,7 @@ from jmbitcoin.secp256k1_ecies import * from jmbitcoin.secp256k1_main import * from jmbitcoin.secp256k1_transaction import * +from collections import Counter SNICKER_MAGIC_BYTES = b'SNICKER' @@ -34,16 +35,23 @@ def snicker_privkey_tweak(priv, tweak): base_priv = secp256k1.PrivateKey(priv) return base_priv.add(tweak).secret + b'\x01' -def verify_snicker_output(tx, pub, tweak, spk_type='p2sh-p2wpkh'): +def verify_snicker_output(tx, pub, tweak, spk_type="p2wpkh"): """ A convenience function to check that one output address in a transaction is a SNICKER-type tweak of an existing key. Returns the index of the output for which this is True (and there must be only 1), and the derived spk, or -1 and None if it is not found exactly once. - TODO Add support for other scriptPubKey types. + Only standard segwit spk types (as used in Joinmarket) are supported. """ assert isinstance(tx, btc.CTransaction) expected_destination_pub = snicker_pubkey_tweak(pub, tweak) - expected_destination_spk = pubkey_to_p2sh_p2wpkh_script(expected_destination_pub) + if spk_type == "p2wpkh": + expected_destination_spk = pubkey_to_p2wpkh_script( + expected_destination_pub) + elif spk_type == "p2sh-p2wpkh": + expected_destination_spk = pubkey_to_p2sh_p2wpkh_script( + expected_destination_pub) + else: + assert False, "JM SNICKER only supports p2sh/p2wpkh" found = 0 for i, o in enumerate(tx.vout): if o.scriptPubKey == expected_destination_spk: @@ -52,3 +60,98 @@ def verify_snicker_output(tx, pub, tweak, spk_type='p2sh-p2wpkh'): if found != 1: return -1, None return found_index, expected_destination_spk + +def construct_snicker_outputs(proposer_input_amount, receiver_input_amount, + receiver_addr, proposer_addr, change_addr, + network_fee, net_transfer): + """ This is abstracted from full SNICKER transaction proposal (see + `jmclient.wallet.SNICKERWalletMixin`) construction, as it is also useful + for making fake SNICKERs. + total_input_amount (int) : value of sum of inputs in sats + receiver_input_amount (int): value of single utxo input of receiver in sats + receiver_addr (str): address for tweaked destination of receiver + proposer_addr (str): address for proposer's coinjoin output + change_addr (str): address for proposer's change output + network_fee (int): bitcoin network transaction fee in sats + net_transfer (int): how much the proposer gives to the receiver in sats + + Returns: + list of outputs, each is of form {"address": x, "value": y} + """ + total_input_amount = proposer_input_amount + receiver_input_amount + total_output_amount = total_input_amount - network_fee + receiver_output_amount = receiver_input_amount + net_transfer + proposer_output_amount = total_output_amount - receiver_output_amount + change_output_amount = total_output_amount - 2 * receiver_output_amount + # callers should only request sane values: + assert all([x>0 for x in [receiver_output_amount, change_output_amount]]) + + # now we must construct the three outputs with correct output amounts. + outputs = [{"address": receiver_addr, "value": receiver_output_amount}] + outputs.append({"address": proposer_addr, "value": receiver_output_amount}) + outputs.append({"address": change_addr, + "value": change_output_amount}) + + return outputs + +def is_snicker_tx(tx, snicker_version=bytes([1])): + """ Returns True if the CTransaction object `tx` + fits the pattern of a SNICKER coinjoin of type + defined in `snicker_version`, or False otherwise. + """ + if not snicker_version == b"\x01": + raise NotImplementedError("Only v1 SNICKER currently implemented.") + return is_snicker_v1_tx(tx) + +def is_snicker_v1_tx(tx): + """ We expect: + * 2 equal outputs, same script type, pubkey hash variant. + * 1 other output (0 is negligible probability hence ignored - if it + was included it would create a lot of false positives). + * >=2 inputs, same script type, pubkey hash variant. + * Input sequence numbers are both 0xffffffff + * nVersion 2 + * nLockTime 0 + The above rules are for matching the v1 variant of SNICKER. + """ + assert isinstance(tx, CTransaction) + if tx.nVersion != 2: + return False + if tx.nLockTime != 0: + return False + if len(tx.vin) < 2: + return False + if len(tx.vout) != 3: + return False + for vi in tx.vin: + if vi.nSequence != 0xffffffff: + return False + # identify if there are two equal sized outs + c = Counter([vo.nValue for vo in tx.vout]) + equal_out = -1 + for x in c: + if c[x] not in [1, 2]: + # note three equal outs technically agrees + # with spec, but negligible prob and will + # create false positives. + return False + if c[x] == 2: + equal_out = x + + if equal_out == -1: + return False + + # ensure that the equal sized outputs have the + # same script type + matched_spk = None + for vo in tx.vout: + if vo.nValue == equal_out: + if not matched_spk: + matched_spk = btc.CCoinAddress.from_scriptPubKey( + vo.scriptPubKey).get_scriptPubKey_type() + else: + if not btc.CCoinAddress.from_scriptPubKey( + vo.scriptPubKey).get_scriptPubKey_type() == matched_spk: + return False + assert matched_spk + return True diff --git a/jmbitcoin/test/test_btc_snicker.py b/jmbitcoin/test/test_btc_snicker.py new file mode 100644 index 0000000..50276fd --- /dev/null +++ b/jmbitcoin/test/test_btc_snicker.py @@ -0,0 +1,96 @@ +import pytest +import copy +import jmbitcoin as btc + +""" Awkward test module name `test_btc_snicker` is to avoid + conflicts with snicker tests in jmclient. +""" + +@pytest.mark.parametrize( + "our_input_val, their_input_val, network_fee, script_type, net_transfer", [ + (24000000, 20000000, 2000, "p2wpkh", 100), + (124000000, 20000000, 800, "p2wpkh", -100), + (24000000, 20000000, 2000, "p2sh-p2wpkh", 100), + (124000000, 20000000, 800, "p2sh-p2wpkh", -100), + ]) +def test_is_snicker_tx(our_input_val, their_input_val, network_fee, + script_type, net_transfer): + our_input = (bytes([1])*32, 0) + their_input = (bytes([2])*32, 1) + assert our_input_val - their_input_val - network_fee > 0 + total_input_amount = our_input_val + their_input_val + total_output_amount = total_input_amount - network_fee + receiver_output_amount = their_input_val + net_transfer + proposer_output_amount = total_output_amount - receiver_output_amount + + # all keys are just made up; only the script type will be checked + privs = [bytes([i])*32 + bytes([1]) for i in range(1,4)] + pubs = [btc.privkey_to_pubkey(x) for x in privs] + + if script_type == "p2wpkh": + spks = [btc.pubkey_to_p2wpkh_script(x) for x in pubs] + elif script_type == "p2sh-p2wpkh": + spks = [btc.pubkey_to_p2sh_p2wpkh_script(x) for x in pubs] + else: + assert False + tweaked_addr, our_addr, change_addr = [str( + btc.CCoinAddress.from_scriptPubKey(x)) for x in spks] + # now we must construct the three outputs with correct output amounts. + outputs = [{"address": tweaked_addr, "value": receiver_output_amount}] + outputs.append({"address": our_addr, "value": receiver_output_amount}) + outputs.append({"address": change_addr, + "value": total_output_amount - 2 * receiver_output_amount}) + assert all([x["value"] > 0 for x in outputs]) + + # make_shuffled_tx mutates ordering (yuck), work with copies only: + outputs1 = copy.deepcopy(outputs) + # version and locktime as currently specified in the BIP + # for 0/1 version SNICKER. (Note the locktime is partly because + # of expected delays). + tx = btc.make_shuffled_tx([our_input, their_input], outputs1, + version=2, locktime=0) + assert btc.is_snicker_tx(tx) + + # construct variants which will be invalid. + + # mixed script types in outputs + wrong_tweaked_spk = btc.pubkey_to_p2pkh_script(pubs[1]) + wrong_tweaked_addr = str(btc.CCoinAddress.from_scriptPubKey( + wrong_tweaked_spk)) + outputs2 = copy.deepcopy(outputs) + outputs2[0] = {"address": wrong_tweaked_addr, + "value": receiver_output_amount} + tx2 = btc.make_shuffled_tx([our_input, their_input], outputs2, + version=2, locktime=0) + assert not btc.is_snicker_tx(tx2) + + # nonequal output amounts + outputs3 = copy.deepcopy(outputs) + outputs3[1] = {"address": our_addr, "value": receiver_output_amount - 1} + tx3 = btc.make_shuffled_tx([our_input, their_input], outputs3, + version=2, locktime=0) + assert not btc.is_snicker_tx(tx3) + + # too few outputs + outputs4 = copy.deepcopy(outputs) + outputs4 = outputs4[:2] + tx4 = btc.make_shuffled_tx([our_input, their_input], outputs4, + version=2, locktime=0) + assert not btc.is_snicker_tx(tx4) + + # too many outputs + outputs5 = copy.deepcopy(outputs) + outputs5.append({"address": change_addr, "value": 200000}) + tx5 = btc.make_shuffled_tx([our_input, their_input], outputs5, + version=2, locktime=0) + assert not btc.is_snicker_tx(tx5) + + # wrong nVersion + tx6 = btc.make_shuffled_tx([our_input, their_input], outputs, + version=1, locktime=0) + assert not btc.is_snicker_tx(tx6) + + # wrong nLockTime + tx7 = btc.make_shuffled_tx([our_input, their_input], outputs, + version=2, locktime=1) + assert not btc.is_snicker_tx(tx7) diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 9b2fd43..2ae09f5 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -24,11 +24,12 @@ from .configure import (load_test_config, process_shutdown, load_program_config, jm_single, get_network, update_persist_config, validate_address, is_burn_destination, get_irc_mchannels, get_blockchain_interface_instance, set_config, is_segwit_mode, - is_native_segwit_mode) + is_native_segwit_mode, JMPluginService) from .blockchaininterface import (BlockchainInterface, RegtestBitcoinCoreInterface, BitcoinCoreInterface) +from .snicker_receiver import SNICKERError, SNICKERReceiver from .client_protocol import (JMTakerClientProtocol, JMClientProtocolFactory, - start_reactor) + start_reactor, SNICKERClientProtocolFactory) from .podle import (set_commitment_file, get_commitment_file, add_external_commitments, PoDLE, generate_podle, get_podle_commitments, @@ -55,7 +56,6 @@ from .wallet_utils import ( from .wallet_service import WalletService from .maker import Maker from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain -from .snicker_receiver import SNICKERError, SNICKERReceiver from .payjoin import (parse_payjoin_setup, send_payjoin, PayjoinServer, JMBIP78ReceiverManager) # Set default logging handler to avoid "No handler found" warnings. diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index a3500cc..91c128d 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -204,10 +204,12 @@ class BitcoinCoreInterface(BlockchainInterface): restart when the connection is healed, but that is tricky). Should not be called directly from outside code. """ + # TODO: flip the logic of this. We almost never want to print these + # out even to debug as they are noisy. if method not in ['importaddress', 'walletpassphrase', 'getaccount', 'gettransaction', 'getrawtransaction', 'gettxout', 'importmulti', 'listtransactions', 'getblockcount', - 'scantxoutset']: + 'scantxoutset', 'getblock', 'getblockhash']: log.debug('rpc: ' + method + " " + str(args)) try: res = self.jsonRpc.call(method, args) @@ -394,6 +396,17 @@ class BitcoinCoreInterface(BlockchainInterface): result.append(result_dict) return result + def get_unspent_indices(self, transaction): + """ Given a CTransaction object, identify the list of + indices of outputs which are unspent (returned as list of ints). + """ + bintxid = transaction.GetTxid()[::-1] + res = self.query_utxo_set([(bintxid, i) for i in range( + len(transaction.vout))]) + # QUS returns 'None' for spent outputs, so filter them out + # and return the indices of the others: + return [i for i, val in enumerate(res) if val] + def estimate_fee_per_kb(self, N): """ The argument N may be either a number of blocks target, for estimation of feerate by Core, or a number of satoshis diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index d69f149..1d89cc7 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -16,27 +16,14 @@ import sys from jmbase import (get_log, EXIT_FAILURE, hextobin, bintohex, utxo_to_utxostr) from jmclient import (jm_single, get_irc_mchannels, - RegtestBitcoinCoreInterface) + RegtestBitcoinCoreInterface, + SNICKERReceiver, process_shutdown) import jmbitcoin as btc jlog = get_log() -class JMProtocolError(Exception): - pass - -class JMClientProtocol(amp.AMP): - def __init__(self, factory, client, nick_priv=None): - self.client = client - self.factory = factory - if not nick_priv: - self.nick_priv = hashlib.sha256( - os.urandom(16)).digest() + b"\x01" - else: - self.nick_priv = nick_priv - - self.shutdown_requested = False - +class BaseClientProtocol(amp.AMP): def checkClientResponse(self, response): """A generic check of client acceptance; any failure is considered criticial. @@ -54,6 +41,146 @@ class JMClientProtocol(amp.AMP): d.addCallback(self.checkClientResponse) d.addErrback(self.defaultErrback) +class JMProtocolError(Exception): + pass + +class SNICKERClientProtocol(BaseClientProtocol): + + def __init__(self, client, servers, tls_whitelist=[], oneshot=False): + # if client is type JMSNICKERReceiver, this will flag + # the use of the receiver workflow (polling loop). + # Otherwise it is assumed to be a proposer workloop, + # which does not have active polling, but only the + # ability to upload when clients call for it. + self.client = client + self.servers = servers + if len(tls_whitelist) == 0: + if isinstance(jm_single().bc_interface, + RegtestBitcoinCoreInterface): + tls_whitelist = ["127.0.0.1"] + self.tls_whitelist = tls_whitelist + self.processed_proposals = [] + self.oneshot = oneshot + + def connectionMade(self): + netconfig = {"socks5_host": jm_single().config.get("PAYJOIN", "onion_socks5_host"), + "socks5_port": jm_single().config.get("PAYJOIN", "onion_socks5_port"), + "servers": self.servers, + "tls_whitelist": ",".join(self.tls_whitelist), + "filterconfig": "", + "credentials": ""} + + if isinstance(self.client, SNICKERReceiver): + d = self.callRemote(commands.SNICKERReceiverInit, + netconfig=json.dumps(netconfig)) + else: + d = self.callRemote(commands.SNICKERProposerInit, + netconfig=json.dumps(netconfig)) + self.defaultCallbacks(d) + + def shutdown(self): + """ Encapsulates shut down actions. + """ + if self.proposal_poll_loop: + self.proposals_poll_loop.stop() + + def poll_for_proposals(self): + """ May be invoked in a LoopingCall or other + event loop. + Retrieves any entries in the proposals_source, then + compares with existing, + and invokes parse_proposal on all new entries. + # TODO considerable thought should go into how to store + proposals cross-runs, and also handling of keys, which + must be optional. + """ + # always check whether the service is still intended to + # be active, before starting the polling actions: + if jm_single().config.get("SNICKER", "enabled") != "true": + self.shutdown() + return + d = self.callRemote(commands.SNICKERReceiverGetProposals) + self.defaultCallbacks(d) + + @commands.SNICKERProposerUp.responder + def on_SNICKER_PROPOSER_UP(self): + jlog.info("SNICKER proposer daemon ready.") + # TODO handle multiple servers correctly + for s in self.servers: + if s == "": + continue + d = self.callRemote(commands.SNICKERRequestPowTarget, + server=s) + self.defaultCallbacks(d) + return {"accepted": True} + + @commands.SNICKERReceivePowTarget.responder + def on_SNICKER_RECEIVE_POW_TARGET(self, server, targetbits): + proposals = self.client.get_proposals(targetbits) + d = self.callRemote(commands.SNICKERProposerPostProposals, + proposals="\n".join([x.decode("utf-8") for x in proposals]), + server = server) + self.defaultCallbacks(d) + return {"accepted": True} + + @commands.SNICKERServerError.responder + def on_SNICKER_SERVER_ERROR(self, server, errorcode): + self.client.info_callback("Server: " + str( + server) + " returned error code: " + str(errorcode)) + return {"accepted": True} + + @commands.SNICKERReceiverUp.responder + def on_SNICKER_RECEIVER_UP(self): + if self.oneshot: + jlog.info("Starting single query to SNICKER server(s).") + reactor.callLater(0.0, self.poll_for_proposals) + else: + jlog.info("Starting SNICKER polling loop") + self.proposal_poll_loop = task.LoopingCall( + self.poll_for_proposals) + poll_interval = int(60.0 * float( + jm_single().config.get("SNICKER", "polling_interval_minutes"))) + self.proposal_poll_loop.start(poll_interval, now=False) + return {"accepted": True} + + @commands.SNICKERReceiverProposals.responder + def on_SNICKER_RECEIVER_PROPOSALS(self, proposals, server): + """ Just passes through the proposals retrieved from + any server, to the SNICKERReceiver client object, asynchronously. + The proposals data must be newline separated. + """ + try: + proposals = proposals.split("\n") + except: + jlog.warn("Error in parsing proposals from server: " + str(server)) + return {"accepted": True} + reactor.callLater(0.0, self.process_proposals, proposals) + return {"accepted": True} + + def process_proposals(self, proposals): + self.client.process_proposals(proposals) + if self.oneshot: + process_shutdown() + + @commands.SNICKERProposalsServerResponse.responder + def on_SNICKER_PROPOSALS_SERVER_RESPONSE(self, response, server): + self.client.info_callback("Response from server: " + str(server) +\ + " was: " + str(response)) + self.client.end_requests_callback(None) + return {"accepted": True} + +class JMClientProtocol(BaseClientProtocol): + def __init__(self, factory, client, nick_priv=None): + self.client = client + self.factory = factory + if not nick_priv: + self.nick_priv = hashlib.sha256( + os.urandom(16)).digest() + b"\x01" + else: + self.nick_priv = nick_priv + + self.shutdown_requested = False + def connectionMade(self): jlog.debug('connection was made, starting client.') self.factory.setClient(self) @@ -499,6 +626,15 @@ class JMTakerClientProtocol(JMClientProtocol): txhex=str(txhex_to_push)) self.defaultCallbacks(d) +class SNICKERClientProtocolFactory(protocol.ClientFactory): + protocol = SNICKERClientProtocol + def buildProtocol(self, addr): + return self.protocol(self.client, self.servers, oneshot=self.oneshot) + def __init__(self, client, servers, oneshot=False): + self.client = client + self.servers = servers + self.oneshot = oneshot + class JMClientProtocolFactory(protocol.ClientFactory): protocol = JMTakerClientProtocol @@ -517,15 +653,17 @@ class JMClientProtocolFactory(protocol.ClientFactory): def buildProtocol(self, addr): return self.protocol(self, self.client) -def start_reactor(host, port, factory, ish=True, daemon=False, rs=True, - gui=False): #pragma: no cover +def start_reactor(host, port, factory=None, snickerfactory=None, ish=True, + daemon=False, rs=True, gui=False): #pragma: no cover #(Cannot start the reactor in tests) #Not used in prod (twisted logging): #startLogging(stdout) - usessl = True if jm_single().config.get("DAEMON", "use_ssl") != 'false' else False + usessl = True if jm_single().config.get("DAEMON", + "use_ssl") != 'false' else False if daemon: try: - from jmdaemon import JMDaemonServerProtocolFactory, start_daemon + from jmdaemon import JMDaemonServerProtocolFactory, start_daemon, \ + SNICKERDaemonServerProtocolFactory except ImportError: jlog.error("Cannot start daemon without jmdaemon package; " "either install it, and restart, or, if you want " @@ -533,6 +671,8 @@ def start_reactor(host, port, factory, ish=True, daemon=False, rs=True, "section of the config. Quitting.") return dfactory = JMDaemonServerProtocolFactory() + if snickerfactory: + sdfactory = SNICKERDaemonServerProtocolFactory() orgport = port while True: try: @@ -546,11 +686,21 @@ def start_reactor(host, port, factory, ish=True, daemon=False, rs=True, jlog.error("Tried 100 ports but cannot listen on any of them. Quitting.") sys.exit(EXIT_FAILURE) port += 1 + if snickerfactory: + start_daemon(host, port-1000, sdfactory, usessl, + './ssl/key.pem', './ssl/cert.pem') + jlog.info("(SNICKER) Listening on port " + str(port-1000)) if usessl: - ctx = ClientContextFactory() - reactor.connectSSL(host, port, factory, ctx) + if factory: + reactor.connectSSL(host, port, factory, ClientContextFactory()) + if snickerfactory: + reactor.connectSSL(host, port-1000, snickerfactory, + ClientContextFactory()) else: - reactor.connectTCP(host, port, factory) + if factory: + reactor.connectTCP(host, port, factory) + if snickerfactory: + reactor.connectTCP(host, port-1000, snickerfactory) if rs: if not gui: reactor.run(installSignalHandlers=ish) diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index b778a48..243bd37 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -397,6 +397,26 @@ minsize = 100000 size_factor = 0.1 gaplimit = 6 + +[SNICKER] + +# any other value than 'true' will be treated as False, +# and no SNICKER actions will be enabled in that case: +enabled = false + +# in satoshis, we require any SNICKER to pay us at least +# this much (can be negative), otherwise we will refuse +# to sign it: +lowest_net_gain = 0 + +# comma separated list of servers (if port is omitted as :port, it +# is assumed to be 80) which we will poll against (all, in sequence); note +# that they are allowed to be *.onion or cleartext servers, and no +# scheme (http(s) etc) needs to be added to the start. +servers = cn5lfwvrswicuxn3gjsxoved6l2gu5hdvwy5l3ev7kg6j7lbji2k7hqd.onion, + +# how many minutes between each polling event to each server above: +polling_interval_minutes = 60 """ #This allows use of the jmclient package with a @@ -466,6 +486,36 @@ def get_config_irc_channel(channel_name): channel += '-sig' return channel +class JMPluginService(object): + """ Allows us to configure on-startup + any additional service (such as SNICKER). + For now only covers logging. + """ + def __init__(self, name, requires_logging=True): + self.name = name + self.requires_logging = requires_logging + + def start_plugin_logging(self, wallet): + """ This requires the name of the active wallet + to set the logfile; TODO other plugin services may + need a different setup. + """ + self.wallet = wallet + self.logfilename = "{}-{}.log".format(self.name, + self.wallet.get_wallet_name()) + self.start_logging() + + def set_log_dir(self, logdirname): + self.logdirname = logdirname + + def start_logging(self): + logFormatter = logging.Formatter( + ('%(asctime)s [%(levelname)-5.5s] {} - %(message)s'.format( + self.name))) + fileHandler = logging.FileHandler( + self.logdirname + '/{}'.format(self.logfilename)) + fileHandler.setFormatter(logFormatter) + get_log().addHandler(fileHandler) def get_network(): """Returns network name""" @@ -510,7 +560,7 @@ def remove_unwanted_default_settings(config): if section.startswith('MESSAGING:'): config.remove_section(section) -def load_program_config(config_path="", bs=None): +def load_program_config(config_path="", bs=None, plugin_services=[]): global_singleton.config.readfp(io.StringIO(defaultconfig)) if not config_path: config_path = lookup_appdata_folder(global_singleton.APPNAME) @@ -608,6 +658,27 @@ def load_program_config(config_path="", bs=None): set_commitment_file(os.path.join(config_path, global_singleton.commit_file_location)) + for p in plugin_services: + # for now, at this config level, the only significance + # of a "plugin" is that it keeps its own separate log. + # We require that a section exists in the config file, + # and that it has enabled=true: + assert isinstance(p, JMPluginService) + if not (global_singleton.config.has_section(p.name) and \ + global_singleton.config.has_option(p.name, "enabled") and \ + global_singleton.config.get(p.name, "enabled") == "true"): + break + if p.requires_logging: + # make sure the environment can accept a logfile by + # creating the directory in the correct place, + # and setting that in the plugin object; the plugin + # itself will switch on its own logging when ready, + # attaching a filehandler to the global log. + plogsdir = os.path.join(os.path.dirname( + global_singleton.config_location), "logs", p.name) + if not os.path.exists(plogsdir): + os.makedirs(plogsdir) + p.set_log_dir(plogsdir) def load_test_config(**kwargs): if "config_path" not in kwargs: diff --git a/jmclient/jmclient/snicker_receiver.py b/jmclient/jmclient/snicker_receiver.py index 62a5079..cdc2f03 100644 --- a/jmclient/jmclient/snicker_receiver.py +++ b/jmclient/jmclient/snicker_receiver.py @@ -1,26 +1,21 @@ #! /usr/bin/env python -import sys - import jmbitcoin as btc from jmclient.configure import jm_single -from jmbase import (get_log, EXIT_FAILURE, utxo_to_utxostr, - bintohex, hextobin) +from jmbase import (get_log, utxo_to_utxostr, + hextobin, bintohex) +from twisted.application.service import Service jlog = get_log() class SNICKERError(Exception): pass -class SNICKERReceiver(object): +class SNICKERReceiver(Service): supported_flags = [] - import_branch = 0 - # TODO implement http api or similar - # for polling, here just a file: - proposals_source = "proposals.txt" - def __init__(self, wallet_service, income_threshold=0, - acceptance_callback=None): + def __init__(self, wallet_service, acceptance_callback=None, + info_callback=None): """ Class to manage processing of SNICKER proposals and co-signs and broadcasts in case the application level @@ -36,9 +31,9 @@ class SNICKERReceiver(object): # The simplest filter on accepting SNICKER joins: # that they pay a minimum of this value in satoshis, - # which can be negative (to account for fees). - # TODO this will be a config variable. - self.income_threshold = income_threshold + # which can be negative (e.g. to account for fees). + self.income_threshold = jm_single().config.getint("SNICKER", + "lowest_net_gain") # The acceptance callback which defines if we accept # a valid proposal and sign it, or not. @@ -47,6 +42,11 @@ class SNICKERReceiver(object): else: self.acceptance_callback = acceptance_callback + # callback for information messages to UI + if not info_callback: + self.info_callback = self.default_info_callback + else: + self.info_callback = info_callback # A list of currently viable key candidates; these must # all be (pub)keys for which the privkey is accessible, # i.e. they must be in-wallet keys. @@ -61,27 +61,11 @@ class SNICKERReceiver(object): # SNICKER transactions in the current run. self.successful_txs = [] - def poll_for_proposals(self): - """ Intended to be invoked in a LoopingCall or other - event loop. - Retrieves any entries in the proposals_source, then - compares with existing, - and invokes parse_proposal on all new entries. - # TODO considerable thought should go into how to store - proposals cross-runs, and also handling of keys, which - must be optional. - """ - new_proposals = [] - with open(self.proposals_source, "rb") as f: - current_entries = f.readlines() - for entry in current_entries: - if entry in self.processed_proposals: - continue - new_proposals.append(entry) - if not self.process_proposals(new_proposals): - jlog.error("Critical logic error, shutting down.") - sys.exit(EXIT_FAILURE) - self.processed_proposals.extend(new_proposals) + # the main monitoring loop that checks for proposals: + self.proposal_poll_loop = None + + def default_info_callback(self, msg): + jlog.info(msg) def default_acceptance_callback(self, our_ins, their_ins, our_outs, their_outs): @@ -99,16 +83,16 @@ class SNICKERReceiver(object): # we use get_all* because for these purposes mixdepth # is irrelevant. utxos = self.wallet_service.get_all_utxos() - print("gau returned these utxos: ", utxos) our_in_amts = [] our_out_amts = [] for i in our_ins: utxo_for_i = (i.prevout.hash[::-1], i.prevout.n) if utxo_for_i not in utxos.keys(): - success, utxostr =utxo_to_utxostr(utxo_for_i) + success, utxostr = utxo_to_utxostr(utxo_for_i) if not success: jlog.error("Code error: input utxo in wrong format.") jlog.debug("The input utxo was not found: " + utxostr) + jlog.debug("NB: This can simply mean the coin is already spent.") return False our_in_amts.append(utxos[utxo_for_i]["value"]) for o in our_outs: @@ -117,8 +101,19 @@ class SNICKERReceiver(object): return False return True + def log_successful_tx(self, tx): + """ TODO: add dedicated SNICKER log file. + """ + self.successful_txs.append(tx) + jlog.info(btc.human_readable_transaction(tx)) + def process_proposals(self, proposals): - """ Each entry in `proposals` is of form: + """ This is the "meat" of the SNICKERReceiver service. + It parses proposals and creates and broadcasts transactions + with the wallet, assuming all conditions are met. + Note that this is ONLY called from the proposals poll loop. + + Each entry in `proposals` is of form: encrypted_proposal - base64 string key - hex encoded compressed pubkey, or '' if the key is not null, we attempt to decrypt and @@ -141,29 +136,31 @@ class SNICKERReceiver(object): """ for kp in proposals: + # handle empty list entries: + if not kp: + continue try: - p, k = kp.split(b',') + p, k = kp.split(',') except: - jlog.error("Invalid proposal string, ignoring: " + kp) + # could argue for info or warning debug level, + # but potential for a lot of unwanted output. + jlog.debug("Invalid proposal string, ignoring: " + kp) + continue if k is not None: # note that this operation will succeed as long as # the key is in the wallet._script_map, which will # be true if the key is at an HD index lower than # the current wallet.index_cache - k = hextobin(k.decode('utf-8')) + k = hextobin(k) addr = self.wallet_service.pubkey_to_addr(k) if not self.wallet_service.is_known_addr(addr): jlog.debug("Key not recognized as part of our " "wallet, ignoring.") continue - # TODO: interface/API of SNICKERWalletMixin would better take - # address as argument here, not privkey: - priv = self.wallet_service.get_key_from_addr(addr) result = self.wallet_service.parse_proposal_to_signed_tx( - priv, p, self.acceptance_callback) + addr, p, self.acceptance_callback) if result[0] is not None: tx, tweak, out_spk = result - # We will: rederive the key as a sanity check, # and see if it matches the claimed spk. # Then, we import the key into the wallet @@ -172,45 +169,53 @@ class SNICKERReceiver(object): # Finally, we co-sign, then push. # (Again, simplest function: checks already passed, # so do it automatically). - # TODO: the more sophisticated actions. tweaked_key = btc.snicker_pubkey_tweak(k, tweak) - tweaked_spk = btc.pubkey_to_p2sh_p2wpkh_script(tweaked_key) + tweaked_spk = self.wallet_service.pubkey_to_script( + tweaked_key) + # Derive original path to make sure we change + # mixdepth: + source_path = self.wallet_service.script_to_path( + self.wallet_service.pubkey_to_script(k)) + # NB This will give the correct source mixdepth independent + # of whether the key is imported or not: + source_mixdepth = self.wallet_service.get_details( + source_path)[0] if not tweaked_spk == out_spk: jlog.error("The spk derived from the pubkey does " "not match the scriptPubkey returned from " "the snicker module - code error.") return False # before import, we should derive the tweaked *private* key - # from the tweak, also: - tweaked_privkey = btc.snicker_privkey_tweak(priv, tweak) - if not btc.privkey_to_pubkey(tweaked_privkey) == tweaked_key: - jlog.error("Was not able to recover tweaked pubkey " - "from tweaked privkey - code error.") - jlog.error("Expected: " + bintohex(tweaked_key)) - jlog.error("Got: " + bintohex(btc.privkey_to_pubkey( - tweaked_privkey))) + # from the tweak, also; failure of this critical sanity check + # is a code error. If the recreated private key matches, we + # import to the wallet. Note that this happens *before* pushing + # the coinjoin transaction to the network, which is advisably + # conservative (never possible to have broadcast a tx without + # having already stored the output's key). + success, msg = self.wallet_service.check_tweak_matches_and_import( + addr, tweak, tweaked_key, source_mixdepth) + if not success: + jlog.error(msg) return False - # the recreated private key matches, so we import to the wallet, - # note that type = None here is because we use the same - # scriptPubKey type as the wallet, this has been implicitly - # checked above by deriving the scriptPubKey. - self.wallet_service.import_private_key(self.import_branch, - self.wallet_service._ENGINE.privkey_to_wif(tweaked_privkey)) - # TODO condition on automatic brdcst or not if not jm_single().bc_interface.pushtx(tx.serialize()): - jlog.error("Failed to broadcast SNICKER CJ.") - return False - self.successful_txs.append(tx) - return True + # this represents an error about state (or conceivably, + # an ultra-short window in which the spent utxo was + # consumed in another transaction), but not really + # an internal logic error, so we do NOT return False + jlog.error("Failed to broadcast SNICKER coinjoin: " +\ + bintohex(tx.GetTxid()[::-1])) + jlog.info(btc.human_readable_transaction(tx)) + jlog.info("Successfully broadcast SNICKER coinjoin: " +\ + bintohex(tx.GetTxid()[::-1])) + self.log_successful_tx(tx) else: jlog.debug('Failed to parse proposal: ' + result[1]) - continue else: # Some extra work to implement checking all possible # keys. - raise NotImplementedError() + jlog.info("Proposal without pubkey was not processed.") # Completed processing all proposals without any logic # errors (whether the proposals were valid or accepted diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 6867fe7..f653ba2 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -5,6 +5,7 @@ import functools import collections import numbers import random +import copy import base64 import json from binascii import hexlify, unhexlify @@ -627,6 +628,24 @@ class BaseWallet(object): mixdepth = self._get_mixdepth_from_path(path) self._utxos.add_utxo(txid, index, path, value, mixdepth, height=height) + def inputs_consumed_by_tx(self, tx): + """ Given a transaction tx, checks + which if any of the inputs belonged to this + wallet, and returns [(index, CTxIn),..] for each. + """ + retval = [] + for i, txin in len(tx.vin): + pub, msg = btc.extract_pubkey_from_witness(tx, i) + if not pub: + # this can certainly occur since other inputs + # may not be a spending a script we recognize; + # so we ignore the msg + continue + script = self.pubkey_to_script(pub) + if script in self._script_map: + retval.append((i, txin)) + return retval + def process_new_tx(self, txd, height=None): """ Given a newly seen transaction, deserialized as CMutableTransaction txd, @@ -1274,16 +1293,45 @@ class SNICKERWalletMixin(object): def __init__(self, storage, **kwargs): super().__init__(storage, **kwargs) - def create_snicker_proposal(self, our_input, their_input, our_input_utxo, + def check_tweak_matches_and_import(self, addr, tweak, tweaked_key, + source_mixdepth): + """ Given the address from our HD wallet, the tweak bytes and + the tweaked public key, check the tweak correctly generates the + claimed tweaked public key. If not, (False, errmsg is returned), + if so, we import the private key to a mixdepth *distinct* from + source_mixdepth, and (True, None) is returned, if it works. + """ + try: + priv = self.get_key_from_addr(addr) + except: + return False, "Not our address" + tweaked_privkey = btc.snicker_privkey_tweak(priv, tweak) + if not btc.privkey_to_pubkey(tweaked_privkey) == tweaked_key: + hextk = bintohex(tweaked_key) + hexftpk = bintohex(btc.privkey_to_pubkey(tweaked_privkey)) + return False, ("Was not able to recover tweaked pubkey " + "from tweaked privkey - code error." + "Expected: " + hextk + ", got: " + hexftpk) + # note that the WIF is not preserving SPK type, it's implied + # for the wallet. + try: + self.import_private_key((source_mixdepth + 1) % (self.mixdepth + 1), + self._ENGINE.privkey_to_wif(tweaked_privkey)) + self.save() + except: + return False, "Failed to import private key." + return True, None + + def create_snicker_proposal(self, our_inputs, their_input, our_input_utxos, their_input_utxo, net_transfer, network_fee, our_priv, their_pub, our_spk, change_spk, encrypted=True, version_byte=1): """ Creates a SNICKER proposal from the given transaction data. This only applies to existing specification, i.e. SNICKER v 00 or 01. This is only to be used for Joinmarket and only segwit wallets. - `our_input`, `their_input` - utxo format used in JM wallets, + `our_inputs`, `their_input` - utxo format used in JM wallets (first is list), keyed by (tixd, n), as dicts (currently of single entry). - `our_input_utxo`, `their..` - type CTxOut (contains value, scriptPubKey) + `our_input_utxos`, `their..` - type CTxOut (contains value, scriptPubKey) net_transfer - amount, after bitcoin transaction fee, transferred from Proposer (our) to Receiver (their). May be negative. network_fee - total bitcoin network transaction fee to be paid (so estimates @@ -1292,7 +1340,7 @@ class SNICKERWalletMixin(object): the tweak as per the BIP. Note `their_pub` may or may not be associated with the input of the receiver, so is specified here separately. Note also that according to the BIP the privkey we use *must* be the one corresponding to - the input we provided, else (properly coded) Receivers will reject our + the first input we provided, else (properly coded) Receivers will reject our proposal. `our_spk` - a scriptPubKey for the Proposer coinjoin output `change_spk` - a change scriptPubkey for the proposer as per BIP @@ -1309,61 +1357,67 @@ class SNICKERWalletMixin(object): assert isinstance(self, PSBTWalletMixin) # before constructing the bitcoin transaction we must calculate the output # amounts - # TODO investigate arithmetic for negative transfer - if our_input_utxo.nValue - their_input_utxo.nValue - network_fee <= 0: + # find our total input value: + our_input_val = sum([x.nValue for x in our_input_utxos]) + if our_input_val - their_input_utxo.nValue - network_fee <= 0: raise Exception( "Cannot create SNICKER proposal, Proposer input too small") - total_input_amount = our_input_utxo.nValue + their_input_utxo.nValue - total_output_amount = total_input_amount - network_fee - receiver_output_amount = their_input_utxo.nValue + net_transfer - proposer_output_amount = total_output_amount - receiver_output_amount - # we must also use ecdh to calculate the output scriptpubkey for the # receiver # First, check that `our_priv` corresponds to scriptPubKey in - # `our_input_utxo` to prevent callers from making useless proposals. + # first entry in `our_input_utxos` to prevent callers from making + # useless proposals. expected_pub = btc.privkey_to_pubkey(our_priv) expected_spk = self.pubkey_to_script(expected_pub) - assert our_input_utxo.scriptPubKey == expected_spk + assert our_input_utxos[0].scriptPubKey == expected_spk # now we create the ecdh based tweak: tweak_bytes = btc.ecdh(our_priv[:-1], their_pub) tweaked_pub = btc.snicker_pubkey_tweak(their_pub, tweak_bytes) - # TODO: remove restriction to one scriptpubkey type - tweaked_spk = btc.pubkey_to_p2sh_p2wpkh_script(tweaked_pub) + wtype = self.get_txtype() + if wtype == "p2wpkh": + tweaked_spk = btc.pubkey_to_p2wpkh_script(tweaked_pub) + elif wtype == "p2sh-p2wpkh": + tweaked_spk = btc.pubkey_to_p2sh_p2wpkh_script(tweaked_pub) + else: + raise NotImplementedError("SNICKER only supports " + "p2sh-p2wpkh or p2wpkh outputs.") tweaked_addr, our_addr, change_addr = [str( btc.CCoinAddress.from_scriptPubKey(x)) for x in ( - tweaked_spk, expected_spk, change_spk)] - # now we must construct the three outputs with correct output amounts. - outputs = [{"address": tweaked_addr, "value": receiver_output_amount}] - outputs.append({"address": our_addr, "value": receiver_output_amount}) - outputs.append({"address": change_addr, - "value": total_output_amount - 2 * receiver_output_amount}) + tweaked_spk, our_spk, change_spk)] + outputs = btc.construct_snicker_outputs(our_input_val, + their_input_utxo.nValue, + tweaked_addr, + our_addr, + change_addr, + network_fee, + net_transfer) assert all([x["value"] > 0 for x in outputs]) # version and locktime as currently specified in the BIP - # for 0/1 version SNICKER. - tx = btc.make_shuffled_tx([our_input, their_input], outputs, + # for 0/1 version SNICKER. (Note the locktime is partly because + # of expected delays). + # We do not use `make_shuffled_tx` because we need a specific order + # on the input side: the receiver's is placed randomly, but the first + # *of ours* is the one used for ECDH. + all_inputs = copy.deepcopy(our_inputs) + insertion_index = random.randrange(len(all_inputs)+1) + all_inputs.insert(insertion_index, their_input) + all_input_utxos = copy.deepcopy(our_input_utxos) + all_input_utxos.insert(insertion_index, their_input_utxo) + # for outputs, we must shuffle as normal: + random.shuffle(outputs) + tx = btc.mktx(all_inputs, outputs, version=2, locktime=0) - # we need to know which randomized input is ours: - our_index = -1 - for i, inp in enumerate(tx.vin): - if our_input == (inp.prevout.hash[::-1], inp.prevout.n): - our_index = i - assert our_index in [0, 1], "code error: our input not in tx" - spent_outs = [our_input_utxo, their_input_utxo] - if our_index == 1: - spent_outs = spent_outs[::-1] # create the psbt and then sign our input. - snicker_psbt = self.create_psbt_from_tx(tx, spent_outs=spent_outs) - + snicker_psbt = self.create_psbt_from_tx(tx, + spent_outs=all_input_utxos) # having created the PSBT, sign our input - # TODO this requires bitcointx updated minor version else fails signed_psbt_and_signresult, err = self.sign_psbt( snicker_psbt.serialize(), with_sign_result=True) assert err is None signresult, partially_signed_psbt = signed_psbt_and_signresult - assert signresult.num_inputs_signed == 1 - assert signresult.num_inputs_final == 1 + assert signresult.num_inputs_signed == len(our_inputs) + assert signresult.num_inputs_final == len(our_inputs) assert not signresult.is_final snicker_serialized_message = btc.SNICKER_MAGIC_BYTES + bytes( [version_byte]) + btc.SNICKER_FLAG_NONE + tweak_bytes + \ @@ -1376,7 +1430,7 @@ class SNICKERWalletMixin(object): # we apply ECIES in the form given by the BIP. return btc.ecies_encrypt(snicker_serialized_message, their_pub) - def parse_proposal_to_signed_tx(self, privkey, proposal, + def parse_proposal_to_signed_tx(self, addr, proposal, acceptance_callback): """ Given a candidate privkey (binary and compressed format), and a candidate encrypted SNICKER proposal, attempt to decrypt @@ -1400,6 +1454,11 @@ class SNICKERWalletMixin(object): """ assert isinstance(self, PSBTWalletMixin) + try: + privkey = self.get_key_from_addr(addr) + except: + return None, "Could not derive privkey from address" + our_pub = btc.privkey_to_pubkey(privkey) if len(proposal) < 5: @@ -1445,16 +1504,16 @@ class SNICKERWalletMixin(object): # which populates the 'is_signed' info fields for us. Note that # we do not use the PSBTWalletMixin.sign_psbt() which automatically # signs with our keys. - if not len(utx.vin) == 2: - return None, "PSBT proposal does not contain 2 inputs." + if len(utx.vin) < 2: + return None, "PSBT proposal does not contain at least 2 inputs." testsignresult = cpsbt.sign(btc.KeyStore(), finalize=False) - print("got sign result: ", testsignresult) + # Note: "num_inputs_signed" refers to how many *we* signed, # which is obviously none here as we provided no keys. if not (testsignresult.num_inputs_signed == 0 and \ - testsignresult.num_inputs_final == 1 and \ + testsignresult.num_inputs_final == len(utx.vin)-1 and \ not testsignresult.is_final): - return None, "PSBT proposal does not contain 1 signature." + return None, "PSBT proposal is not fully signed by proposer." # Validate that we own one SNICKER style output: spk = btc.verify_snicker_output(utx, our_pub, tweak_bytes) @@ -1477,12 +1536,17 @@ class SNICKERWalletMixin(object): # To allow the acceptance callback to assess validity, we must identify # which input is ours and which is(are) not. - # TODO This check may (will) change if we allow non-p2sh-pwpkh inputs: + # See https://github.com/Simplexum/python-bitcointx/issues/39 + # for background on why this is different per wallet type unsigned_index = -1 for i, psbtinputsigninfo in enumerate(testsignresult.inputs_info): if psbtinputsigninfo is None: unsigned_index = i break + if psbtinputsigninfo.num_new_sigs == 0 and \ + psbtinputsigninfo.is_final == False: + unsigned_index = i + break assert unsigned_index != -1 # All validation checks passed. We now check whether the #transaction is acceptable according to the caller: @@ -1499,7 +1563,7 @@ class SNICKERWalletMixin(object): return None, "Unable to sign proposed PSBT, reason: " + err signresult, signed_psbt = signresult_and_signedpsbt assert signresult.num_inputs_signed == 1 - assert signresult.num_inputs_final == 2 + assert signresult.num_inputs_final == len(utx.vin) assert signresult.is_final # we now know the transaction is valid and fully signed; return to caller, # along with supporting data for this tx: diff --git a/jmclient/jmclient/wallet_service.py b/jmclient/jmclient/wallet_service.py index ea55c98..99fbd1f 100644 --- a/jmclient/jmclient/wallet_service.py +++ b/jmclient/jmclient/wallet_service.py @@ -663,6 +663,45 @@ class WalletService(Service): self.wallet.set_next_index(mixdepth, address_type, highest_used_index + 1) + def get_all_transactions(self): + """ Get all transactions (spending or receiving) that + are currently recorded by our blockchain interface as relating + to this wallet, as a list. + """ + res = [] + processed_txids = [] + for r in self.bci._yield_transactions( + self.get_wallet_name()): + txid = r["txid"] + if txid not in processed_txids: + tx = self.bci.get_transaction(hextobin(txid)) + res.append(self.bci.get_deser_from_gettransaction(tx)) + processed_txids.append(txid) + return res + + def get_transaction(self, txid): + """ If the transaction for txid is an in-wallet + transaction, will return a CTransaction object for it; + if not, will return None. + """ + tx = self.bci.get_transaction(txid) + if not tx: + return None + return self.bci.get_deser_from_gettransaction(tx) + + def get_block_height(self, blockhash): + return self.bci.get_block_height(blockhash) + + def get_transaction_block_height(self, tx): + """ Given a CTransaction object tx, return + the block height at which it was mined, or False + if it is not found in the blockchain (including if + it is not a wallet tx and so can't be queried). + """ + txid = tx.GetTxid()[::-1] + return self.get_block_height(self.bci.get_transaction( + txid)["blockhash"]) + def sync_addresses(self): """ Triggered by use of --recoversync option in scripts, attempts a full scan of the blockchain without assuming diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index 15d8773..43622c9 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -7,9 +7,10 @@ import abc from twisted.python.log import startLogging from optparse import OptionParser from jmbase import get_log -from jmclient import Maker, jm_single, load_program_config, \ - JMClientProtocolFactory, start_reactor, calc_cj_fee, \ - WalletService, add_base_options +from jmclient import (Maker, jm_single, load_program_config, + JMClientProtocolFactory, start_reactor, calc_cj_fee, + WalletService, add_base_options, SNICKERReceiver, + SNICKERClientProtocolFactory) from .wallet_utils import open_test_wallet_maybe, get_wallet_path from jmbase.support import EXIT_ARGERROR, EXIT_FAILURE @@ -310,11 +311,17 @@ def ygmain(ygclass, nickserv_password='', gaplimit=6): cjfee_factor, size_factor]) jlog.info('starting yield generator') clientfactory = JMClientProtocolFactory(maker, proto_type="MAKER") - + if jm_single().config.get("SNICKER", "enabled") == "true": + snicker_r = SNICKERReceiver(wallet_service) + servers = jm_single().config.get("SNICKER", "servers").split(",") + snicker_factory = SNICKERClientProtocolFactory(snicker_r, servers) + else: + snicker_factory = None nodaemon = jm_single().config.getint("DAEMON", "no_daemon") daemon = True if nodaemon == 1 else False if jm_single().config.get("BLOCKCHAIN", "network") in ["regtest", "testnet", "signet"]: startLogging(sys.stdout) start_reactor(jm_single().config.get("DAEMON", "daemon_host"), jm_single().config.getint("DAEMON", "daemon_port"), - clientfactory, daemon=daemon) + clientfactory, snickerfactory=snicker_factory, + daemon=daemon) diff --git a/jmclient/test/test_snicker.py b/jmclient/test/test_snicker.py index 017f70b..230b6df 100644 --- a/jmclient/test/test_snicker.py +++ b/jmclient/test/test_snicker.py @@ -11,7 +11,6 @@ from jmbase import get_log, bintohex from jmclient import (load_test_config, estimate_tx_fee, SNICKERReceiver, direct_send, SegwitLegacyWallet, BaseWallet) -TEST_PROPOSALS_FILE = "test_proposals.txt" log = get_log() @pytest.mark.parametrize( @@ -33,8 +32,7 @@ def test_snicker_e2e(setup_snicker, nw, wallet_structures, """ # TODO: Make this test work with native segwit wallets - wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt, - wallet_cls=SegwitLegacyWallet) + wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt) for w in wallets.values(): w['wallet'].sync_wallet(fast=True) print(wallets) @@ -82,7 +80,8 @@ def test_snicker_e2e(setup_snicker, nw, wallet_structures, their_input = (txid1, txid1_index) our_input_utxo = btc.CMutableTxOut(prop_utxo['value'], prop_utxo['script']) - fee_est = estimate_tx_fee(len(tx.vin), 2) + fee_est = estimate_tx_fee(len(tx.vin), 2, + txtype=wallet_p.get_txtype()) change_spk = wallet_p.get_new_script(0, BaseWallet.ADDRESS_TYPE_INTERNAL) encrypted_proposals = [] @@ -92,8 +91,8 @@ def test_snicker_e2e(setup_snicker, nw, wallet_structures, # not just one guessed output, if desired. encrypted_proposals.append( wallet_p.create_snicker_proposal( - our_input, their_input, - our_input_utxo, + [our_input], their_input, + [our_input_utxo], tx.vout[txid1_index], net_transfer, fee_est, @@ -102,11 +101,8 @@ def test_snicker_e2e(setup_snicker, nw, wallet_structures, prop_utxo['script'], change_spk, version_byte=1) + b"," + bintohex(p).encode('utf-8')) - with open(TEST_PROPOSALS_FILE, "wb") as f: - f.write(b"\n".join(encrypted_proposals)) sR = SNICKERReceiver(wallet_r) - sR.proposals_source = TEST_PROPOSALS_FILE # avoid clashing with mainnet - sR.poll_for_proposals() + sR.process_proposals([x.decode("utf-8") for x in encrypted_proposals]) assert len(sR.successful_txs) == 1 wallet_r.process_new_tx(sR.successful_txs[0]) end_utxos = wallet_r.get_all_utxos() @@ -117,7 +113,3 @@ def test_snicker_e2e(setup_snicker, nw, wallet_structures, @pytest.fixture(scope="module") def setup_snicker(request): load_test_config() - def teardown(): - if os.path.exists(TEST_PROPOSALS_FILE): - os.remove(TEST_PROPOSALS_FILE) - request.addfinalizer(teardown) diff --git a/jmdaemon/jmdaemon/__init__.py b/jmdaemon/jmdaemon/__init__.py index 1b92812..0af37ff 100644 --- a/jmdaemon/jmdaemon/__init__.py +++ b/jmdaemon/jmdaemon/__init__.py @@ -9,7 +9,7 @@ from .message_channel import MessageChannel, MessageChannelCollection from .orderbookwatch import OrderbookWatch from jmbase import commands from .daemon_protocol import (JMDaemonServerProtocolFactory, JMDaemonServerProtocol, - start_daemon) + start_daemon, SNICKERDaemonServerProtocolFactory) from .protocol import (COMMAND_PREFIX, ORDER_KEYS, NICK_HASH_LENGTH, NICK_MAX_ENCODED, JM_VERSION, JOINMARKET_NICK_HEADER) from .message_channel import MessageChannelCollection diff --git a/jmdaemon/jmdaemon/daemon_protocol.py b/jmdaemon/jmdaemon/daemon_protocol.py index bf8f1bb..062f054 100644 --- a/jmdaemon/jmdaemon/daemon_protocol.py +++ b/jmdaemon/jmdaemon/daemon_protocol.py @@ -9,14 +9,20 @@ from .protocol import (COMMAND_PREFIX, ORDER_KEYS, NICK_HASH_LENGTH, COMMITMENT_PREFIXES) from .irc import IRCMessageChannel -from jmbase import hextobin +from jmbase import (hextobin, is_hs_uri, get_tor_agent, + get_nontor_agent, BytesProducer, wrapped_urlparse) from jmbase.commands import * from twisted.protocols import amp from twisted.internet import reactor, ssl from twisted.internet.protocol import ServerFactory from twisted.internet.error import (ConnectionLost, ConnectionAborted, ConnectionClosed, ConnectionDone) +from twisted.web.http_headers import Headers +from twisted.web.client import ResponseFailed, readBody +from txtorcon.socks import HostUnreachableError from twisted.python import log +import urllib.parse as urlparse +from urllib.parse import urlencode import json import threading import os @@ -87,6 +93,186 @@ def check_utxo_blacklist(commitment, persist=False): class JMProtocolError(Exception): pass +class HTTPPassThrough(amp.AMP): + """ This class supports passing through + requests over HTTPS or over a socks proxy to a remote + onion service, or multiple. + """ + + def on_INIT(self, netconfig): + """ The network config must be passed in json + and contains these fields: + socks5_host + socks5_proxy + servers (comma separated list) + tls_whitelist (comma separated list) + filterconfig (not yet defined) + credentials (not yet defined) + """ + netconfig = json.loads(netconfig) + self.socks5_host = netconfig["socks5_host"] + self.socks5_port = int(netconfig["socks5_port"]) + self.servers = [a for a in netconfig["servers"] if a != ""] + self.tls_whitelist = [a for a in netconfig["tls_whitelist"].split( + ",") if a != ""] + + def getAgentDestination(self, server, params=None): + tor_url_data = is_hs_uri(server) + if tor_url_data: + # note: SSL over Tor not supported at the moment: + agent = get_tor_agent(self.socks5_host, self.socks5_port) + else: + agent = get_nontor_agent(self.tls_whitelist) + + destination_url = server.encode("utf-8") + url_parts = list(wrapped_urlparse(destination_url)) + if params: + url_parts[4] = urlencode(params).encode("utf-8") + destination_url = urlparse.urlunparse(url_parts) + return (agent, destination_url) + + def getRequest(self, server, success_callback, url=None, headers=None): + """ Make GET request to server server, if response received OK, + passed to success_callback, which must have function signature + (response, server). + """ + agent, destination_url = self.getAgentDestination(server) + if url: + destination_url = destination_url + url + # Deliberately sending NO headers; this could be a tricky point + # for anonymity of users, as much boilerplate code will not create + # requests that look like this. + headers = Headers({}) if not headers else headers + d = agent.request(b"GET", destination_url, headers) + d.addCallback(success_callback, server) + # note that the errback (here "noResponse") is *not* triggered + # by a server rejection (which is accompanied by a non-200 + # status code returned), but by failure to communicate. + def noResponse(failure): + failure.trap(ResponseFailed, ConnectionRefusedError, + HostUnreachableError, ConnectionLost) + log.msg(failure.value) + d.addErrback(noResponse) + + def postRequest(self, body, server, success_callback, headers=None): + """ Pass body of post request as string, will be encoded here. + """ + agent, destination_url = self.getAgentDestination(server) + body = BytesProducer(body.encode("utf-8")) + d = agent.request(b"POST", destination_url, + Headers({}), bodyProducer=body) + d.addCallback(success_callback, server) + # note that the errback (here "noResponse") is *not* triggered + # by a server rejection (which is accompanied by a non-200 + # status code returned), but by failure to communicate. + def noResponse(failure): + failure.trap(ResponseFailed, ConnectionRefusedError, + HostUnreachableError, ConnectionLost) + log.msg(failure.value) + self.callRemote(SNICKERProposalsServerResponse, + response="failure to connect", + server=server) + d.addErrback(noResponse) + + def checkClientResponse(self, response): + """A generic check of client acceptance; any failure + is considered criticial. + """ + if 'accepted' not in response or not response['accepted']: + reactor.stop() #pragma: no cover + + def defaultErrback(self, failure): + """TODO better network error handling. + """ + failure.trap(ConnectionAborted, ConnectionClosed, + ConnectionDone, ConnectionLost) + + def defaultCallbacks(self, d): + d.addCallback(self.checkClientResponse) + d.addErrback(self.defaultErrback) + +class SNICKERDaemonServerProtocol(HTTPPassThrough): + + @SNICKERProposerPostProposals.responder + def on_SNICKER_PROPOSER_POST_PROPOSALS(self, proposals, server): + """ Receives a list of proposals to be posted to a specific + server. + """ + self.postRequest(proposals, server, self.receive_proposals_response) + return {"accepted": True} + + def receive_proposals_response(self, response, server): + d = readBody(response) + if int(response.code) != 200: + log.msg("Server returned error code: " + str(response.code)) + d = self.callRemote(SNICKERServerError, server=server, + errorcode=response.code) + self.defaultCallbacks(d) + return + d.addCallback(self.process_proposals_response_from_server, server) + + @SNICKERReceiverInit.responder + def on_SNICKER_RECEIVER_INIT(self, netconfig): + self.on_INIT(netconfig) + d = self.callRemote(SNICKERReceiverUp) + self.defaultCallbacks(d) + return {'accepted': True} + + @SNICKERProposerInit.responder + def on_SNICKER_PROPOSER_INIT(self, netconfig): + self.on_INIT(netconfig) + d = self.callRemote(SNICKERProposerUp) + self.defaultCallbacks(d) + return {"accepted": True} + + @SNICKERReceiverGetProposals.responder + def on_SNICKER_RECEIVER_GET_PROPOSALS(self): + for server in self.servers: + self.getRequest(server, self.receive_proposals_from_server) + return {'accepted': True} + + def receive_proposals_from_server(self, response, server): + """ Parses the response from one server. + """ + # if the response code is not 200 OK, we must let the client + # know that this server is not responding as expected. + if int(response.code) != 200: + d = self.callRemote(SNICKERServerError, + server=server, + errorcode = response.code) + self.defaultCallbacks(d) + return + d = readBody(response) + d.addCallback(self.process_proposals_from_server, server) + + @SNICKERRequestPowTarget.responder + def on_SNICKER_REQUEST_POW_TARGET(self, server): + self.getRequest(server, self.receive_pow_target, + url=b"/target") + return {"accepted": True} + + def receive_pow_target(self, response, server): + d = readBody(response) + d.addCallback(self.process_pow_target, server) + + def process_pow_target(self, response_body, server): + d = self.callRemote(SNICKERReceivePowTarget, + server=server, + targetbits=int(response_body.decode("utf-8"))) + self.defaultCallbacks(d) + + def process_proposals_from_server(self, response, server): + d = self.callRemote(SNICKERReceiverProposals, + proposals=response.decode("utf-8"), + server=server) + self.defaultCallbacks(d) + + def process_proposals_response_from_server(self, response_body, server): + d = self.callRemote(SNICKERProposalsServerResponse, + response=response_body.decode("utf-8"), + server=server) + self.defaultCallbacks(d) + class JMDaemonServerProtocol(amp.AMP, OrderbookWatch): def __init__(self, factory): @@ -647,6 +833,9 @@ class JMDaemonServerProtocolFactory(ServerFactory): def buildProtocol(self, addr): return JMDaemonServerProtocol(self) +class SNICKERDaemonServerProtocolFactory(ServerFactory): + protocol = SNICKERDaemonServerProtocol + def start_daemon(host, port, factory, usessl=False, sslkey=None, sslcert=None): if usessl: assert sslkey diff --git a/scripts/joinmarketd.py b/scripts/joinmarketd.py index 9292178..14f3f9c 100755 --- a/scripts/joinmarketd.py +++ b/scripts/joinmarketd.py @@ -14,7 +14,8 @@ def startup_joinmarketd(host, port, usessl, factories=None, """ startLogging(sys.stdout) if not factories: - factories = [jmdaemon.JMDaemonServerProtocolFactory(),] + factories = [jmdaemon.JMDaemonServerProtocolFactory(), + jmdaemon.SNICKERDaemonServerProtocolFactory()] for factory in factories: jmdaemon.start_daemon(host, port, factory, usessl, './ssl/key.pem', './ssl/cert.pem') diff --git a/scripts/snicker/create-snicker-proposal.py b/scripts/snicker/create-snicker-proposal.py new file mode 100644 index 0000000..50e150d --- /dev/null +++ b/scripts/snicker/create-snicker-proposal.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 + +description="""A rudimentary implementation of creation of a SNICKER proposal. + +**THIS TOOL DOES NOT SCAN FOR CANDIDATE TRANSACTIONS** + +It only creates proposals on candidate transactions (individually) +that you have already found. + +Input: the user's wallet, mixdepth to source their (1) coin from, +and a hex encoded pre-existing bitcoin transaction (fully signed) +as target. +User chooses the input to source the pubkey from, and the output +to use to create the SNICKER coinjoin. Tx fees are sourced from +the config, and the user specifies interactively the number of sats +to award the receiver (can be negative). + +Once the proposal is created, it is uploaded to the servers as per +the `servers` setting in `joinmarket.cfg`, unless the -n option is +specified (see help for options), in which case the proposal is +output to stdout in the same string format: base64proposal,hexpubkey. +""" + +import sys +from optparse import OptionParser +from jmbase import BytesProducer, bintohex, jmprint, hextobin, \ + EXIT_ARGERROR, EXIT_FAILURE, EXIT_SUCCESS, get_pow +import jmbitcoin as btc +from jmclient import (RegtestBitcoinCoreInterface, process_shutdown, + jm_single, load_program_config, check_regtest, select_one_utxo, + estimate_tx_fee, SNICKERReceiver, add_base_options, get_wallet_path, + open_test_wallet_maybe, WalletService, SNICKERClientProtocolFactory, + start_reactor, JMPluginService) +from jmclient.configure import get_log + +log = get_log() + +def main(): + parser = OptionParser( + usage= + 'usage: %prog [options] walletname hex-tx input-index output-index net-transfer', + description=description + ) + add_base_options(parser) + parser.add_option('-m', + '--mixdepth', + action='store', + type='int', + dest='mixdepth', + help='mixdepth/account to spend from, default=0', + default=0) + parser.add_option( + '-g', + '--gap-limit', + action='store', + type='int', + dest='gaplimit', + default = 6, + help='gap limit for Joinmarket wallet, default 6.' + ) + parser.add_option( + '-n', + '--no-upload', + action='store_true', + dest='no_upload', + default=False, + help="if set, we don't upload the new proposal to the servers" + ) + parser.add_option( + '-f', + '--txfee', + action='store', + type='int', + dest='txfee', + default=-1, + help='Bitcoin miner tx_fee to use for transaction(s). A number higher ' + 'than 1000 is used as "satoshi per KB" tx fee. A number lower than that ' + 'uses the dynamic fee estimation of your blockchain provider as ' + 'confirmation target. This temporarily overrides the "tx_fees" setting ' + 'in your joinmarket.cfg. Works the same way as described in it. Check ' + 'it for examples.') + parser.add_option('-a', + '--amtmixdepths', + action='store', + type='int', + dest='amtmixdepths', + help='number of mixdepths in wallet, default 5', + default=5) + (options, args) = parser.parse_args() + snicker_plugin = JMPluginService("SNICKER") + load_program_config(config_path=options.datadir, + plugin_services=[snicker_plugin]) + if len(args) != 5: + jmprint("Invalid arguments, see --help") + sys.exit(EXIT_ARGERROR) + wallet_name, hextx, input_index, output_index, net_transfer = args + input_index, output_index, net_transfer = [int(x) for x in [ + input_index, output_index, net_transfer]] + check_regtest() + + # If tx_fees are set manually by CLI argument, override joinmarket.cfg: + if int(options.txfee) > 0: + jm_single().config.set("POLICY", "tx_fees", str(options.txfee)) + max_mix_depth = max([options.mixdepth, options.amtmixdepths - 1]) + wallet_path = get_wallet_path(wallet_name, None) + wallet = open_test_wallet_maybe( + wallet_path, wallet_name, max_mix_depth, + wallet_password_stdin=options.wallet_password_stdin, + gap_limit=options.gaplimit) + wallet_service = WalletService(wallet) + if wallet_service.rpc_error: + sys.exit(EXIT_FAILURE) + snicker_plugin.start_plugin_logging(wallet_service) + # in this script, we need the wallet synced before + # logic processing for some paths, so do it now: + while not wallet_service.synced: + wallet_service.sync_wallet(fast=not options.recoversync) + # the sync call here will now be a no-op: + wallet_service.startService() + + # now that the wallet is available, we can construct a proposal + # before encrypting it: + originating_tx = btc.CMutableTransaction.deserialize(hextobin(hextx)) + txid1 = originating_tx.GetTxid()[::-1] + # the proposer wallet needs to choose a single utxo, from his selected + # mixdepth, that is bigger than the output amount of tx1 at the given + # index. + fee_est = estimate_tx_fee(2, 3, txtype=wallet_service.get_txtype()) + amt_required = originating_tx.vout[output_index].nValue + fee_est + + prop_utxo_dict = wallet_service.select_utxos(options.mixdepth, + amt_required) + prop_utxos = list(prop_utxo_dict) + prop_utxo_vals = [prop_utxo_dict[x] for x in prop_utxos] + # get the private key for that utxo + priv = wallet_service.get_key_from_addr( + wallet_service.script_to_addr(prop_utxo_vals[0]['script'])) + # construct the arguments for the snicker proposal: + our_input_utxos = [btc.CMutableTxOut(x['value'], + x['script']) for x in prop_utxo_vals] + + # destination must be a different mixdepth: + prop_destn_spk = wallet_service.get_new_script(( + options.mixdepth + 1) % (wallet_service.mixdepth + 1), 1) + change_spk = wallet_service.get_new_script(options.mixdepth, 1) + their_input = (txid1, output_index) + # we also need to extract the pubkey of the chosen input from + # the witness; we vary this depending on our wallet type: + pubkey, msg = btc.extract_pubkey_from_witness(originating_tx, input_index) + if not pubkey: + log.error("Failed to extract pubkey from transaction: {}".format(msg)) + sys.exit(EXIT_FAILURE) + encrypted_proposal = wallet_service.create_snicker_proposal( + prop_utxos, their_input, + our_input_utxos, + originating_tx.vout[output_index], + net_transfer, + fee_est, + priv, + pubkey, + prop_destn_spk, + change_spk, + version_byte=1) + b"," + bintohex(pubkey).encode('utf-8') + if options.no_upload: + jmprint(encrypted_proposal.decode("utf-8")) + sys.exit(EXIT_SUCCESS) + + nodaemon = jm_single().config.getint("DAEMON", "no_daemon") + daemon = True if nodaemon == 1 else False + snicker_client = SNICKERPostingClient([encrypted_proposal]) + servers = jm_single().config.get("SNICKER", "servers").split(",") + snicker_pf = SNICKERClientProtocolFactory(snicker_client, servers) + start_reactor(jm_single().config.get("DAEMON", "daemon_host"), + jm_single().config.getint("DAEMON", "daemon_port"), + None, snickerfactory=snicker_pf, + daemon=daemon) + +class SNICKERPostingClient(object): + """ A client object which stores proposals + ready to be sent to the server/servers, and appends + proof of work to them according to the server's rules. + """ + def __init__(self, pre_nonce_proposals, info_callback=None, + end_requests_callback=None): + # the encrypted proposal without the nonce appended for PoW + self.pre_nonce_proposals = pre_nonce_proposals + + self.proposals_with_nonce = [] + + # callback for conveying information messages + if not info_callback: + self.info_callback = self.default_info_callback + else: + self.info_callback = info_callback + + # callback for action at the end of a set of + # submissions to servers; by default, this + # is "one-shot"; we submit to all servers in the + # config, then shut down the script. + if not end_requests_callback: + self.end_requests_callback = \ + self.default_end_requests_callback + + def default_end_requests_callback(self, response): + process_shutdown() + + def default_info_callback(self, msg): + jmprint(msg) + + def get_proposals(self, targetbits): + # the data sent to the server is base64encryptedtx,key,nonce; the nonce + # part is generated in get_pow(). + for p in self.pre_nonce_proposals: + nonceval, preimage, niter = get_pow(p+b",", nbits=targetbits, + truncate=32) + log.debug("Got POW preimage: {}".format(preimage.decode("utf-8"))) + if nonceval is None: + log.error("Failed to generate proof of work, message:{}".format( + preimage)) + sys.exit(EXIT_FAILURE) + self.proposals_with_nonce.append(preimage) + return self.proposals_with_nonce + +if __name__ == "__main__": + main() + jmprint('done', "success") diff --git a/scripts/snicker/receive-snicker.py b/scripts/snicker/receive-snicker.py new file mode 100644 index 0000000..cd4403d --- /dev/null +++ b/scripts/snicker/receive-snicker.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 + +from optparse import OptionParser +import sys +from jmbase import get_log, jmprint +from jmclient import (jm_single, load_program_config, WalletService, + open_test_wallet_maybe, get_wallet_path, + check_regtest, add_base_options, start_reactor, + SNICKERClientProtocolFactory, SNICKERReceiver, + JMPluginService) +from jmbase.support import EXIT_ARGERROR + +jlog = get_log() + +def receive_snicker_main(): + usage = """ Use this script to receive proposals for SNICKER +coinjoins, parse them and then broadcast coinjoins +that fit your criteria. See the SNICKER section of +joinmarket.cfg to set your criteria. +The only argument to this script is the (JM) wallet +file against which to check. +Once all proposals have been parsed, the script will +quit. +Usage: %prog [options] wallet file [proposal] +""" + parser = OptionParser(usage=usage) + add_base_options(parser) + parser.add_option('-g', '--gap-limit', action='store', type="int", + dest='gaplimit', default=6, + help='gap limit for wallet, default=6') + parser.add_option('-m', '--mixdepth', action='store', type='int', + dest='mixdepth', default=0, + help="mixdepth to source coins from") + parser.add_option('-a', + '--amtmixdepths', + action='store', + type='int', + dest='amtmixdepths', + help='number of mixdepths in wallet, default 5', + default=5) + parser.add_option( + '-n', + '--no-upload', + action='store_true', + dest='no_upload', + default=False, + help="if set, we read the proposal from the command line" + ) + + (options, args) = parser.parse_args() + if len(args) < 1: + parser.error('Needs a wallet file as argument') + sys.exit(EXIT_ARGERROR) + wallet_name = args[0] + snicker_plugin = JMPluginService("SNICKER") + load_program_config(config_path=options.datadir, + plugin_services=[snicker_plugin]) + + check_regtest() + + wallet_path = get_wallet_path(wallet_name, None) + max_mix_depth = max([options.mixdepth, options.amtmixdepths - 1]) + wallet = open_test_wallet_maybe( + wallet_path, wallet_name, max_mix_depth, + wallet_password_stdin=options.wallet_password_stdin, + gap_limit=options.gaplimit) + wallet_service = WalletService(wallet) + snicker_plugin.start_plugin_logging(wallet_service) + while not wallet_service.synced: + wallet_service.sync_wallet(fast=not options.recoversync) + wallet_service.startService() + + nodaemon = jm_single().config.getint("DAEMON", "no_daemon") + daemon = True if nodaemon == 1 else False + snicker_r = SNICKERReceiver(wallet_service) + if options.no_upload: + proposal = args[1] + snicker_r.process_proposals([proposal]) + return + servers = jm_single().config.get("SNICKER", "servers").split(",") + snicker_pf = SNICKERClientProtocolFactory(snicker_r, servers, oneshot=True) + start_reactor(jm_single().config.get("DAEMON", "daemon_host"), + jm_single().config.getint("DAEMON", "daemon_port"), + None, snickerfactory=snicker_pf, + daemon=daemon) + +if __name__ == "__main__": + receive_snicker_main() + jmprint('done') diff --git a/scripts/snicker/snicker-finder.py b/scripts/snicker/snicker-finder.py new file mode 100644 index 0000000..4fc2dee --- /dev/null +++ b/scripts/snicker/snicker-finder.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +description="""Find SNICKER candidate transactions on +the blockchain. + +Using a connection to Bitcoin Core, which allows retrieving +full blocks, this script will list the transaction IDs of +transactions that fit the pattern of SNICKER, as codified in +https://gist.github.com/AdamISZ/2c13fb5819bd469ca318156e2cf25d79 +and as checked in the `jmbitcoin.snicker` module function +`is_snicker_tx`, and also optionally, transactions that fit the +pattern of Joinmarket coinjoins (see -j). + +Pass a starting and finishing block value as argument. If the +finishing block is not provided, it is assumed to be the latest +block. + +**Note that this is slow.** + +This script does *NOT* require a wallet, but it does require +a connection to Core, so does not work with `no-blockchain`. +Note that this script obviates the need to have txindex enabled +in Bitcoin Core in order to get full transactions, since it +parses the raw blocks. +""" + +from optparse import OptionParser +from jmbase import bintohex, EXIT_ARGERROR, EXIT_FAILURE, jmprint +import jmbitcoin as btc +from jmclient import (jm_single, add_base_options, load_program_config, + check_regtest) +from jmclient.configure import get_log + +log = get_log() + +def found_str(ttype, tx, b): + return "Found {} transaction: {} in block: {}".format( + ttype, bintohex(tx.GetTxid()[::-1]), b) + +def write_candidate_to_file(ttype, candidate, blocknum, unspents, filename): + """ Appends the details for the candidate + transaction to the chosen textfile. + """ + with open(filename, "a") as f: + f.write(found_str(ttype, candidate, blocknum) + "\n") + f.write(btc.human_readable_transaction(candidate)+"\n") + f.write("Full transaction hex for creating a proposal is " + "found in the above.\n") + f.write("The unspent indices are: " + " ".join( + (str(u) for u in unspents)) + "\n") +def main(): + parser = OptionParser( + usage= + 'usage: %prog [options] startingblock [endingblock]', + description=description + ) + add_base_options(parser) + parser.add_option('-f', + '--filename', + action='store', + type='str', + dest='candidate_file_name', + help='filename to write details of candidate ' + 'transactions, default ./candidates.txt', + default='candidates.txt') + parser.add_option( + '-j', + '--include-jm', + action='store_true', + dest='include_joinmarket', + default=True, + help="scan for Joinmarket coinjoin outputs, as well as SNICKER.") + + (options, args) = parser.parse_args() + load_program_config(config_path=options.datadir) + if len(args) not in [1,2]: + log.error("Invalid arguments, see --help") + sys.exit(EXIT_ARGERROR) + + startblock = int(args[0]) + if len(args) == 1: + endblock = jm_single().bc_interface.get_current_block_height() + else: + endblock = int(args[1]) + + check_regtest() + + for b in range(startblock, endblock + 1): + block = jm_single().bc_interface.get_block(b) + for t in btc.get_transactions_in_block(block): + if btc.is_snicker_tx(t): + log.info(found_str("SNICKER", t, b)) + # get list of unspent outputs; if empty, skip, + # otherwise, persist to candidate file with unspents + # marked. + unspents = jm_single().bc_interface.get_unspent_indices(t) + if len(unspents) == 0: + continue + write_candidate_to_file("SNICKER", t, b, unspents, + options.candidate_file_name) + # note elif avoids wasting computation if we already found SNICKER: + elif options.include_joinmarket: + cj_amount, n = btc.is_jm_tx(t) + # here we don't care about the stats; the tx is printed anyway. + if cj_amount: + log.info(found_str("Joinmarket coinjoin", t, b)) + unspents = jm_single().bc_interface.get_unspent_indices(t) + if len(unspents) == 0: + continue + write_candidate_to_file("Joinmarket coinjoin", t, b, + unspents, options.candidate_file_name) + log.info("Finished processing block: {}".format(b)) +if __name__ == "__main__": + main() + jmprint('done', "success") diff --git a/scripts/snicker/snicker-recovery.py b/scripts/snicker/snicker-recovery.py new file mode 100644 index 0000000..c5c8e1e --- /dev/null +++ b/scripts/snicker/snicker-recovery.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 + +description="""This tool is to be used in a case where +a user has a BIP39 seedphrase but has no wallet file and no +backup of imported keys, and they had earlier used SNICKER. + +This will usually not be needed as you should keep a backup +of your *.jmdat joinmarket wallet file, which contains all +this information. + +Before using this tool, you need to do: +`python wallet-tool.py recover` to recover the wallet from +seed, and then: +`bitcoin-cli rescanblockchain ...` +for an appropriate range of blocks in order for Bitcoin Core +to get a record of the transactions that happened with your +HD addresses. + +Then, you can run this script to find all the SNICKER-generated +imported addresses that either did have, or still do have, keys +and have them imported back into the wallet. +(Note that this of course won't find any other non-SNICKER imported +keys, so as a reminder, *always* back up either jmdat wallet files, +or at least, the imported keys themselves.) +""" + +from optparse import OptionParser +from jmbase import bintohex, EXIT_ARGERROR, EXIT_FAILURE, jmprint +import jmbitcoin as btc +from jmclient import (jm_single, add_base_options, load_program_config, + check_regtest, get_wallet_path, open_test_wallet_maybe, + WalletService) +from jmclient.configure import get_log + +log = get_log() + +def get_pubs_and_indices_of_inputs(tx, wallet_service, ours): + """ Returns a list of items (pubkey, index), + one per input at index index, in transaction + tx, spending pubkey pubkey, if the input is ours + if ours is True, else returns the complementary list. + """ + our_ins = [] + not_our_ins = [] + for i in range(len(tx.vin)): + pub, msg = btc.extract_pubkey_from_witness(tx, i) + if not pub: + continue + if not wallet_service.is_known_script( + wallet_service.pubkey_to_script(pub)): + not_our_ins.append((pub, i)) + else: + our_ins.append((pub, i)) + if ours: + return our_ins + else: + return not_our_ins + +def get_pubs_and_indices_of_ancestor_inputs(txin, wallet_service, ours): + """ For a transaction input txin, retrieve the spent transaction, + and iterate over its inputs, returning a list of items + (pubkey, index) all of which belong to us if ours is True, + or else the complementary set. + Note: the ancestor transactions must be in the dict txlist, which is + keyed by txid and values are CTransaction; if not, + an error occurs. This is assumed to be the case because all ancestors + must be either in the set returned by wallet_sync, or else in the set + of SNICKER transactions found so far. + """ + tx = wallet_service.get_transaction(txin.prevout.hash[::-1]) + return get_pubs_and_indices_of_inputs(tx, wallet_service, ours=ours) + +def main(): + parser = OptionParser( + usage= + 'usage: %prog [options] walletname', + description=description + ) + parser.add_option('-m', '--mixdepth', action='store', type='int', + dest='mixdepth', default=0, + help="mixdepth to source coins from") + parser.add_option('-a', + '--amtmixdepths', + action='store', + type='int', + dest='amtmixdepths', + help='number of mixdepths in wallet, default 5', + default=5) + parser.add_option('-g', + '--gap-limit', + type="int", + action='store', + dest='gaplimit', + help='gap limit for wallet, default=6', + default=6) + add_base_options(parser) + (options, args) = parser.parse_args() + load_program_config(config_path=options.datadir) + check_regtest() + if len(args) != 1: + log.error("Invalid arguments, see --help") + sys.exit(EXIT_ARGERROR) + wallet_name = args[0] + wallet_path = get_wallet_path(wallet_name, None) + max_mix_depth = max([options.mixdepth, options.amtmixdepths - 1]) + wallet = open_test_wallet_maybe( + wallet_path, wallet_name, max_mix_depth, + wallet_password_stdin=options.wallet_password_stdin, + gap_limit=options.gaplimit) + wallet_service = WalletService(wallet) + + # step 1: do a full recovery style sync. this will pick up + # all addresses that we expect to match transactions against, + # from a blank slate Core wallet that originally had no imports. + if not options.recoversync: + jmprint("Recovery sync was not set, but using it anyway.") + while not wallet_service.synced: + wallet_service.sync_wallet(fast=False) + # Note that the user may be interrupted above by the rescan + # request; this is as for normal scripts; after the rescan is done + # (usually, only once, but, this *IS* needed here, unlike a normal + # wallet generation event), we just try again. + + # Now all address from HD are imported, we need to grab + # all the transactions for those addresses; this includes txs + # that *spend* as well as receive our coins, so will include + # "first-out" SNICKER txs as well as ordinary spends and JM coinjoins. + seed_transactions = wallet_service.get_all_transactions() + + # Search for SNICKER txs and add them if they match. + # We proceed recursively; we find all one-out matches, then + # all 2-out matches, until we find no new ones and stop. + + if len(seed_transactions) == 0: + jmprint("No transactions were found for this wallet. Did you rescan?") + return False + + new_txs = [] + current_block_heights = set() + for tx in seed_transactions: + if btc.is_snicker_tx(tx): + jmprint("Found a snicker tx: {}".format(bintohex(tx.GetTxid()[::-1]))) + equal_outs = btc.get_equal_outs(tx) + if not equal_outs: + continue + if all([wallet_service.is_known_script( + x.scriptPubKey) == False for x in [a[1] for a in equal_outs]]): + # it is now *very* likely that one of the two equal + # outputs is our SNICKER custom output + # script; notice that in this case, the transaction *must* + # have spent our inputs, since it didn't recognize ownership + # of either coinjoin output (and if it did recognize the change, + # it would have recognized the cj output also). + # We try to regenerate one of the outputs, but warn if + # we can't. + my_indices = get_pubs_and_indices_of_inputs(tx, wallet_service, ours=True) + for mypub, mi in my_indices: + for eo in equal_outs: + for (other_pub, i) in get_pubs_and_indices_of_inputs(tx, wallet_service, ours=False): + for (our_pub, j) in get_pubs_and_indices_of_ancestor_inputs(tx.vin[mi], wallet_service, ours=True): + our_spk = wallet_service.pubkey_to_script(our_pub) + our_priv = wallet_service.get_key_from_addr( + wallet_service.script_to_addr(our_spk)) + tweak_bytes = btc.ecdh(our_priv[:-1], other_pub) + tweaked_pub = btc.snicker_pubkey_tweak(our_pub, tweak_bytes) + tweaked_spk = wallet_service.pubkey_to_script(tweaked_pub) + if tweaked_spk == eo[1].scriptPubKey: + # TODO wallet.script_to_addr has a dubious assertion, that's why + # we use btc method directly: + address_found = str(btc.CCoinAddress.from_scriptPubKey(btc.CScript(tweaked_spk))) + #address_found = wallet_service.script_to_addr(tweaked_spk) + jmprint("Found a new SNICKER output belonging to us.") + jmprint("Output address {} in the following transaction:".format( + address_found)) + jmprint(btc.human_readable_transaction(tx)) + jmprint("Importing the address into the joinmarket wallet...") + # NB for a recovery we accept putting any imported keys all into + # the same mixdepth (0); TODO investigate correcting this, it will + # be a little complicated. + success, msg = wallet_service.check_tweak_matches_and_import(wallet_service.script_to_addr(our_spk), + tweak_bytes, tweaked_pub, wallet_service.mixdepth) + if not success: + jmprint("Failed to import SNICKER key: {}".format(msg), "error") + return False + else: + jmprint("... success.") + # we want the blockheight to track where the next-round rescan + # must start from + current_block_heights.add(wallet_service.get_transaction_block_height(tx)) + # add this transaction to the next round. + new_txs.append(tx) + if len(new_txs) == 0: + return True + seed_transactions.extend(new_txs) + earliest_new_blockheight = min(current_block_heights) + jmprint("New SNICKER addresses were imported to the Core wallet; " + "do rescanblockchain again, starting from block {}, before " + "restarting this script.".format(earliest_new_blockheight)) + return False + +if __name__ == "__main__": + res = main() + if not res: + jmprint("Script finished, recovery is NOT complete.", level="warning") + else: + jmprint("Script finished, recovery is complete.") diff --git a/scripts/snicker/snicker-seed-tx.py b/scripts/snicker/snicker-seed-tx.py new file mode 100644 index 0000000..a51855b --- /dev/null +++ b/scripts/snicker/snicker-seed-tx.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 + +description="""Make fake SNICKER transactions to aid discovery. + +Use this script to send money to yourself in a transaction which +fits the format of SNICKER v1 (so it will have two equal sized +outputs and a change output, also obeying the other minor rules +for SNICKER). + +Having done this your transaction will be picked up by blockchain +scanners looking for the SNICKER "fingerprint", allowing them +to propose coinjoins with your coins. + +The transaction is generated with at least TWO utxos from your chosen + source mixdepth/account (-m), so it must contain at least two. +The reason for using one account, not two, is to prevent violating +the principle of not co-spending from different accounts; even though +this is a simulated coinjoin, it may be deducible that it is only really +a *signalling* fake coinjoin, so it is better not to violate the principle. +""" + +import sys +import random +from optparse import OptionParser +from jmbase import BytesProducer, bintohex, jmprint, hextobin, \ + EXIT_ARGERROR, EXIT_FAILURE, EXIT_SUCCESS +import jmbitcoin as btc +from jmclient import (RegtestBitcoinCoreInterface, process_shutdown, + jm_single, load_program_config, check_regtest, select_one_utxo, + estimate_tx_fee, SNICKERReceiver, add_base_options, get_wallet_path, + open_test_wallet_maybe, WalletService, SNICKERClientProtocolFactory, + start_reactor, JMPluginService) +from jmclient.support import select_greedy, NotEnoughFundsException +from jmclient.configure import get_log + +log = get_log() + +def main(): + parser = OptionParser( + usage= + 'usage: %prog [options] walletname', + description=description + ) + add_base_options(parser) + parser.add_option('-m', + '--mixdepth', + action='store', + type='int', + dest='mixdepth', + help='mixdepth/account, default 0', + default=0) + parser.add_option( + '-g', + '--gap-limit', + action='store', + type='int', + dest='gaplimit', + default = 6, + help='gap limit for Joinmarket wallet, default 6.' + ) + parser.add_option( + '-f', + '--txfee', + action='store', + type='int', + dest='txfee', + default=-1, + help='Bitcoin miner tx_fee to use for transaction(s). A number higher ' + 'than 1000 is used as "satoshi per KB" tx fee. A number lower than that ' + 'uses the dynamic fee estimation of your blockchain provider as ' + 'confirmation target. This temporarily overrides the "tx_fees" setting ' + 'in your joinmarket.cfg. Works the same way as described in it. Check ' + 'it for examples.') + parser.add_option('-a', + '--amtmixdepths', + action='store', + type='int', + dest='amtmixdepths', + help='number of mixdepths in wallet, default 5', + default=5) + parser.add_option('-N', + '--net-transfer', + action='store', + type='int', + dest='net_transfer', + help='how many sats are sent to the "receiver", default randomised.', + default=-1000001) + (options, args) = parser.parse_args() + snicker_plugin = JMPluginService("SNICKER") + load_program_config(config_path=options.datadir, + plugin_services=[snicker_plugin]) + if len(args) != 1: + log.error("Invalid arguments, see --help") + sys.exit(EXIT_ARGERROR) + wallet_name = args[0] + check_regtest() + # If tx_fees are set manually by CLI argument, override joinmarket.cfg: + if int(options.txfee) > 0: + jm_single().config.set("POLICY", "tx_fees", str(options.txfee)) + max_mix_depth = max([options.mixdepth, options.amtmixdepths - 1]) + wallet_path = get_wallet_path(wallet_name, None) + wallet = open_test_wallet_maybe( + wallet_path, wallet_name, max_mix_depth, + wallet_password_stdin=options.wallet_password_stdin, + gap_limit=options.gaplimit) + wallet_service = WalletService(wallet) + if wallet_service.rpc_error: + sys.exit(EXIT_FAILURE) + snicker_plugin.start_plugin_logging(wallet_service) + # in this script, we need the wallet synced before + # logic processing for some paths, so do it now: + while not wallet_service.synced: + wallet_service.sync_wallet(fast=not options.recoversync) + # the sync call here will now be a no-op: + wallet_service.startService() + fee_est = estimate_tx_fee(2, 3, txtype=wallet_service.get_txtype()) + + # first, order the utxos in the mixepth by size. Then (this is the + # simplest algorithm; we could be more sophisticated), choose the + # *second* largest utxo as the receiver utxo; this ensures that we + # have enough for the proposer to cover. We consume utxos greedily, + # meaning we'll at least some of the time, be consolidating. + utxo_dict = wallet_service.get_utxos_by_mixdepth()[options.mixdepth] + if not len(utxo_dict) >= 2: + log.error("Cannot create fake SNICKER tx without at least two utxos, quitting") + sys.exit(EXIT_ARGERROR) + # sort utxos by size + sorted_utxos = sorted(list(utxo_dict.keys()), + key=lambda k: utxo_dict[k]['value'], + reverse=True) + # receiver is the second largest: + receiver_utxo = sorted_utxos[1] + receiver_utxo_val = utxo_dict[receiver_utxo] + # gather the other utxos into a list to select from: + nonreceiver_utxos = [sorted_utxos[0]] + sorted_utxos[2:] + # get the net transfer in our fake coinjoin: + if options.net_transfer < -1000001: + log.error("Net transfer must be greater than negative 1M sats") + sys.exit(EXIT_ARGERROR) + if options.net_transfer == -1000001: + # default; low-ish is more realistic and avoids problems + # with dusty utxos + options.net_transfer = random.randint(-1000, 1000) + + # select enough to cover: receiver value + fee + transfer + breathing room + # we select relatively greedily to support consolidation, since + # this transaction does not pretend to isolate the coins. + try: + available = [{'utxo': utxo, 'value': utxo_dict[utxo]["value"]} + for utxo in nonreceiver_utxos] + # selection algos return [{"utxo":..,"value":..}]: + prop_utxos = {x["utxo"] for x in select_greedy(available, + receiver_utxo_val["value"] + fee_est + options.net_transfer + 1000)} + prop_utxos = list(prop_utxos) + prop_utxo_vals = [utxo_dict[prop_utxo] for prop_utxo in prop_utxos] + except NotEnoughFundsException as e: + log.error(repr(e)) + sys.exit(EXIT_FAILURE) + + # Due to the fake nature of this transaction, and its distinguishability + # (not only in trivial output pattern, but also in subset-sum), there + # is little advantage in making it use different output mixdepths, so + # here to prevent fragmentation, everything is kept in the same mixdepth. + receiver_addr, proposer_addr, change_addr = (wallet_service.script_to_addr( + wallet_service.get_new_script(options.mixdepth, 1)) for _ in range(3)) + # persist index update: + wallet_service.save_wallet() + outputs = btc.construct_snicker_outputs( + sum([x["value"] for x in prop_utxo_vals]), + receiver_utxo_val["value"], + receiver_addr, + proposer_addr, + change_addr, + fee_est, + options.net_transfer) + tx = btc.make_shuffled_tx(prop_utxos + [receiver_utxo], + outputs, + version=2, + locktime=0) + # before signing, check we satisfied the criteria, otherwise + # this is pointless! + if not btc.is_snicker_tx(tx): + log.error("Code error, created non-SNICKER tx, not signing.") + sys.exit(EXIT_FAILURE) + + # sign all inputs + # scripts: {input_index: (output_script, amount)} + our_inputs = {} + for index, ins in enumerate(tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) + script = utxo_dict[utxo]['script'] + amount = utxo_dict[utxo]['value'] + our_inputs[index] = (script, amount) + success, msg = wallet_service.sign_tx(tx, our_inputs) + if not success: + log.error("Failed to sign transaction: " + msg) + sys.exit(EXIT_FAILURE) + # TODO condition on automatic brdcst or not + if not jm_single().bc_interface.pushtx(tx.serialize()): + # this represents an error about state (or conceivably, + # an ultra-short window in which the spent utxo was + # consumed in another transaction), but not really + # an internal logic error, so we do NOT return False + log.error("Failed to broadcast fake SNICKER coinjoin: " +\ + bintohex(tx.GetTxid()[::-1])) + log.info(btc.human_readable_transaction(tx)) + sys.exit(EXIT_FAILURE) + log.info("Successfully broadcast fake SNICKER coinjoin: " +\ + bintohex(tx.GetTxid()[::-1])) + +if __name__ == "__main__": + main() + jmprint('done', "success") diff --git a/scripts/snicker/snicker-server.py b/scripts/snicker/snicker-server.py new file mode 100644 index 0000000..5f6677b --- /dev/null +++ b/scripts/snicker/snicker-server.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 + +""" +A rudimentary implementation of a server, allowing POST of proposals +in base64 format, with POW attached required, +and GET of all current proposals, for SNICKER. +Serves only over Tor onion service. +For persistent onion services, specify public port, local port and +hidden service directory: + +`python snicker-server.py 80 7080 /my/hiddenservicedir` + +... and (a) make sure these settings match those in your Tor config, +and also (b) note that the hidden service hostname may not be displayed +if the running user, understandably, do not have permissions to read that +directory. + +If you only want an ephemeral onion service, for testing, just run without +arguments: + +`python snicker-server.py` + +""" + +from twisted.internet import reactor +from twisted.internet.defer import Deferred +from twisted.web.server import Site +from twisted.web.resource import Resource +from twisted.internet.endpoints import TCP4ClientEndpoint, UNIXClientEndpoint, serverFromString +import txtorcon +import sys +import base64 +import json +import sqlite3 +import threading +import hashlib +from io import BytesIO +from jmbase import jmprint, hextobin, bintohex, verify_pow +from jmclient import process_shutdown, jm_single, load_program_config +from jmclient.configure import get_log + +# Note: this is actually a duplication of the +# string in jmbitcoin.secp256k1_ecies, but this is deliberate, +# as we want this tool to have no dependency on jmbitcoin. +ECIES_MAGIC_BYTES = b'BIE1' + +log = get_log() + +database_file_name = "proposals.db" +database_table_name = "proposals" + +class SNICKERServer(Resource): + # rudimentary: flat file, TODO location of file + DATABASE = "snicker-proposals.txt" + + def __init__(self): + self.dblock = threading.Lock() + self.conn = sqlite3.connect(database_file_name, check_same_thread=False) + # TODO: ? + #con.row_factory = dict_factory + + self.cursor = self.conn.cursor() + try: + self.dblock.acquire(True) + # note the pubkey is *NOT* a primary key, by + # design; we need to be able to create multiple + # proposals against one key. + self.cursor.execute("CREATE TABLE IF NOT EXISTS {}(" + "pubkey TEXT NOT NULL, proposal TEXT NOT NULL, " + "unique (pubkey, proposal));".format(database_table_name)) + finally: + self.dblock.release() + + # initial PoW setting; todo, change this: + self.set_pow_target_bits(8) + self.nonce_length = 10 + super().__init__() + + isLeaf = True + + def set_pow_target_bits(self, nbits): + self.nbits = nbits + + def get_pow_target_bits(self): + return self.nbits + + def return_error(self, request, error_meaning, + error_code="unavailable", http_code=400): + """ + We return, to the sender, stringified json in the body as per the above. + """ + request.setResponseCode(http_code) + request.setHeader(b"content-type", b"text/html; charset=utf-8") + log.debug("Returning an error: " + str( + error_code) + ": " + str(error_meaning)) + return json.dumps({"errorCode": error_code, + "message": error_meaning}).encode("utf-8") + + def render_GET(self, request): + """GET request to "/" retrieves the entire current data set. + GET "/target" retrieves the current nbits target for PoW. + It's intended that proposers request the target in real time + before each submission, so that the server can dynamically update + it at any time. + """ + log.debug("GET request, path: {}".format(request.path)) + if request.path == b"/target": + return self.serve_pow_target(request) + if request.path != b"/": + return self.return_error(request, "Invalid request path", + "invalid-request-path") + proposals = self.get_all_current_proposals() + request.setHeader(b"content-length", + ("%d" % len(proposals)).encode("ascii")) + return proposals.encode("ascii") + + def serve_pow_target(self, request): + targetbits = ("%d" % self.nbits).encode("ascii") + request.setHeader(b"content-length", + ("%d" % len(targetbits)).encode("ascii")) + return targetbits + + def render_POST(self, request): + """ An individual proposal may be submitted in base64, with key + appended after newline separator in hex. + """ + log.debug("The server got this POST request: ") + # unfortunately the twisted Request object is not + # easily serialized: + log.debug(request) + log.debug(request.method) + log.debug(request.uri) + log.debug(request.args) + sender_parameters = request.args + log.debug(request.path) + # defer logging of raw request content: + proposals = request.content + if not isinstance(proposals, BytesIO): + return self.return_error(request, "Invalid request format", + "invalid-request-format") + proposals = proposals.read() + # for now, only allowing proposals of form "base64ciphertext,hexkey", + #newline separated: + proposals = proposals.split(b"\n") + log.debug("Client send proposal list of length: " + str( + len(proposals))) + accepted_proposals = [] + for proposal in proposals: + if len(proposal) == 0: + continue + try: + encryptedtx, key, nonce = proposal.split(b",") + bin_key = hextobin(key.decode('utf-8')) + bin_nonce = hextobin(nonce.decode('utf-8')) + base64.b64decode(encryptedtx) + except: + log.warn("This proposal was not accepted: " + proposal.decode( + "utf-8")) + # give up immediately in case of format error: + return self.return_error(request, "Invalid request format", + "invalid-request-format") + if not verify_pow(proposal, nbits=self.nbits, truncate=32): + return self.return_error(request, "Insufficient PoW", + "insufficient proof of work") + accepted_proposals.append((key, encryptedtx)) + + # the proposals are valid format-wise; add them to the database + for p in accepted_proposals: + # note we will ignore errors here and continue; + # warning will be shown in logs from called fn. + self.add_proposal(p) + content = "{} proposals accepted".format(len(accepted_proposals)) + request.setHeader(b"content-length", ("%d" % len(content)).encode( + "ascii")) + return content.encode("ascii") + + def add_proposal(self, p): + proposal_to_add = tuple(x.decode("utf-8") for x in p) + try: + self.cursor.execute('INSERT INTO {} VALUES(?, ?);'.format( + database_table_name),proposal_to_add) + except sqlite3.Error as e: + log.warn("Error inserting data into table: {}".format( + " ".join(e.args))) + return False + self.conn.commit() + return True + + def dbquery(self, querystr, params, return_results=False): + try: + self.dblock.acquire(True) + if return_results: + return self.cursor.execute( + querystr, params).fetchall() + self.cursor.execute(querystr, params) + finally: + self.dblock.release() + + def get_all_keys(self): + rows = self.dbquery('SELECT DISTINCT pubkey FROM {};'.format( + database_table_name), (), True) + if not rows: + return [] + return list([x[0] for x in rows]) + + @classmethod + def db_row_to_proposal_string(cls, row): + assert len(row) == 2 + key, proposal = row + return proposal + "," + key + + def get_all_current_proposals(self): + rows = self.dbquery('SELECT * from {};'.format( + database_table_name), (), True) + return "\n".join([self.db_row_to_proposal_string(x) for x in rows]) + + def get_proposals_for_key(self, key): + rows = self.dbquery('SELECT proposal FROM {} WHERE pubkey=?'.format( + database_table_name), (key,), True) + if not rows: + return [] + return rows + +class SNICKERServerManager(object): + + def __init__(self, port, local_port=None, + hsdir=None, + control_port=9051, + uri_created_callback=None, + info_callback=None, + shutdown_callback=None): + # port is the *public* port, default 80 + # if local_port is None, we follow the process + # to create an ephemeral hidden service. + # if local_port is a valid port, we start the + # hidden service configured at directory hsdir. + # In the latter case, note the patch described at + # https://github.com/meejah/txtorcon/issues/347 is required. + self.port = port + self.local_port = local_port + if self.local_port is not None: + assert hsdir is not None + self.hsdir = hsdir + self.control_port = control_port + if not uri_created_callback: + self.uri_created_callback = self.default_info_callback + else: + self.uri_created_callback = uri_created_callback + if not info_callback: + self.info_callback = self.default_info_callback + else: + self.info_callback = info_callback + + self.shutdown_callback =shutdown_callback + + def default_info_callback(self, msg): + jmprint(msg) + + def start_snicker_server_and_tor(self): + """ Packages the startup of the receiver side. + """ + self.server = SNICKERServer() + self.site = Site(self.server) + self.site.displayTracebacks = False + jmprint("Attempting to start onion service on port: " + str( + self.port) + " ...") + self.start_tor() + + def setup_failed(self, arg): + errmsg = "Setup failed: " + str(arg) + log.error(errmsg) + self.info_callback(errmsg) + process_shutdown() + + def create_onion_ep(self, t): + if self.local_port: + endpointString = "onion:{}:controlPort={}:localPort={}:hiddenServiceDir={}".format( + self.port, self.control_port,self.local_port, self.hsdir) + return serverFromString(reactor, endpointString) + else: + # ephemeral onion: + self.tor_connection = t + return t.create_onion_endpoint(self.port, version=3) + + def onion_listen(self, onion_ep): + return onion_ep.listen(self.site) + + def print_host(self, ep): + """ Callback fired once the HS is available; + receiver user needs a BIP21 URI to pass to + the sender: + """ + self.info_callback("Your hidden service is available: ") + # Note that ep,getHost().onion_port must return the same + # port as we chose in self.port; if not there is an error. + assert ep.getHost().onion_port == self.port + self.uri_created_callback(str(ep.getHost().onion_uri)) + + def start_tor(self): + """ This function executes the workflow + of starting the hidden service. + """ + if not self.local_port: + control_host = jm_single().config.get("PAYJOIN", "tor_control_host") + control_port = int(jm_single().config.get("PAYJOIN", "tor_control_port")) + if str(control_host).startswith('unix:'): + control_endpoint = UNIXClientEndpoint(reactor, control_host[5:]) + else: + control_endpoint = TCP4ClientEndpoint(reactor, control_host, control_port) + d = txtorcon.connect(reactor, control_endpoint) + d.addCallback(self.create_onion_ep) + d.addErrback(self.setup_failed) + else: + d = Deferred() + d.callback(None) + d.addCallback(self.create_onion_ep) + # TODO: add errbacks to the next two calls in + # the chain: + d.addCallback(self.onion_listen) + d.addCallback(self.print_host) + + def shutdown(self): + self.tor_connection.protocol.transport.loseConnection() + process_shutdown(self.mode) + self.info_callback("Hidden service shutdown complete") + if self.shutdown_callback: + self.shutdown_callback() + +def snicker_server_start(port, local_port=None, hsdir=None): + ssm = SNICKERServerManager(port, local_port=local_port, hsdir=hsdir) + ssm.start_snicker_server_and_tor() + +if __name__ == "__main__": + load_program_config(bs="no-blockchain") + # in testing, we can optionally use ephemeral; + # in testing or prod we can use persistent: + if len(sys.argv) < 2: + snicker_server_start(80) + else: + port = int(sys.argv[1]) + local_port = int(sys.argv[2]) + hsdir = sys.argv[3] + snicker_server_start(port, local_port, hsdir) + reactor.run() diff --git a/test/ygrunner.py b/test/ygrunner.py index 6264e0c..a21ddf7 100644 --- a/test/ygrunner.py +++ b/test/ygrunner.py @@ -17,7 +17,7 @@ import random from jmbase import jmprint from jmclient import YieldGeneratorBasic, load_test_config, jm_single,\ JMClientProtocolFactory, start_reactor, SegwitWallet,\ - SegwitLegacyWallet, cryptoengine + SegwitLegacyWallet, cryptoengine, SNICKERClientProtocolFactory, SNICKERReceiver class MaliciousYieldGenerator(YieldGeneratorBasic): @@ -169,12 +169,19 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt, if malicious: yg.set_maliciousness(malicious, mtype="tx") clientfactory = JMClientProtocolFactory(yg, proto_type="MAKER") + if jm_single().config.get("SNICKER", "enabled") == "true": + snicker_r = SNICKERReceiver(wallet_service_yg) + servers = jm_single().config.get("SNICKER", "servers").split(",") + snicker_factory = SNICKERClientProtocolFactory(snicker_r, servers) + else: + snicker_factory = None nodaemon = jm_single().config.getint("DAEMON", "no_daemon") daemon = True if nodaemon == 1 else False rs = True if i == num_ygs - 1 else False start_reactor(jm_single().config.get("DAEMON", "daemon_host"), jm_single().config.getint("DAEMON", "daemon_port"), - clientfactory, daemon=daemon, rs=rs) + clientfactory, snickerfactory=snicker_factory, + daemon=daemon, rs=rs) @pytest.fixture(scope="module") def setup_ygrunner():