From 613bab145d269c21383d23514d187207c7119f97 Mon Sep 17 00:00:00 2001 From: bitromortac Date: Fri, 19 Feb 2021 08:20:46 +0100 Subject: [PATCH] mpp_split: implement splitting logic --- electrum/mpp_split.py | 227 +++++++++++++++++++++++++++++++ electrum/tests/test_mpp_split.py | 75 ++++++++++ 2 files changed, 302 insertions(+) create mode 100644 electrum/mpp_split.py create mode 100644 electrum/tests/test_mpp_split.py diff --git a/electrum/mpp_split.py b/electrum/mpp_split.py new file mode 100644 index 000000000..d0a1d60a5 --- /dev/null +++ b/electrum/mpp_split.py @@ -0,0 +1,227 @@ +import random +from typing import List, Tuple, Optional, Sequence, Dict +from collections import defaultdict +from .util import profiler +from .lnutil import NoPathFound + +PART_PENALTY = 1.0 # 1.0 results in avoiding splits +MIN_PART_MSAT = 10_000_000 # we don't want to split indefinitely + +# these parameters determine the granularity of the newly suggested configurations +REDISTRIBUTION_FRACTION = 10 +SPLIT_FRACTION = 10 + +# these parameters affect the computational work in the probabilistic algorithm +STARTING_CONFIGS = 30 +CANDIDATES_PER_LEVEL = 20 +REDISTRIBUTE = 5 + + +def unique_hierarchy(hierarchy: Dict[int, List[Dict[bytes, int]]]) -> Dict[int, List[Dict[bytes, int]]]: + new_hierarchy = defaultdict(list) + for number_parts, configs in hierarchy.items(): + unique_configs = set() + for config in configs: + # config dict can be out of order, so sort, otherwise not unique + unique_configs.add(tuple((c, config[c]) for c in sorted(config.keys()))) + for unique_config in unique_configs: + new_hierarchy[number_parts].append( + {t[0]: t[1] for t in unique_config}) + return new_hierarchy + + +def number_nonzero_parts(configuration: Dict[bytes, int]): + return len([v for v in configuration.values() if v]) + + +def create_starting_split_hierarchy(amount_msat: int, channels_with_funds: Dict[bytes, int]): + """Distributes the amount to send to a single or more channels in several + ways (randomly).""" + # TODO: find all possible starting configurations deterministically + # could try all permutations + + split_hierarchy = defaultdict(list) + channels_order = list(channels_with_funds.keys()) + + for _ in range(STARTING_CONFIGS): + # shuffle to have different starting points + random.shuffle(channels_order) + + configuration = {} + amount_added = 0 + for c in channels_order: + s = channels_with_funds[c] + if amount_added == amount_msat: + configuration[c] = 0 + else: + amount_to_add = amount_msat - amount_added + amt = min(s, amount_to_add) + configuration[c] = amt + amount_added += amt + if amount_added != amount_msat: + raise NoPathFound("Channels don't have enough sending capacity.") + split_hierarchy[number_nonzero_parts(configuration)].append(configuration) + + return unique_hierarchy(split_hierarchy) + + +def balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): + check = ( + proposed_balance_to < MIN_PART_MSAT or + proposed_balance_to > channels_with_funds[channel_to] or + proposed_balance_from < MIN_PART_MSAT or + proposed_balance_from > channels_with_funds[channel_from] + ) + return check + + +def propose_new_configuration(channels_with_funds: Dict[bytes, int], configuration: Dict[bytes, int], + amount_msat: int, preserve_number_parts=True) -> Dict[bytes, int]: + """Randomly alters a split configuration. If preserve_number_parts, the + configuration stays within the same class of number of splits.""" + + # there are three basic operations to reach different split configurations: + # redistribute, split, swap + + def redistribute(config: dict): + # we redistribute the amount from a nonzero channel to a nonzero channel + redistribution_amount = amount_msat // REDISTRIBUTION_FRACTION + nonzero = [ck for ck, cv in config.items() if + cv >= redistribution_amount] + if len(nonzero) == 1: # we only have a single channel, so we can't redistribute + return config + + channel_from = random.choice(nonzero) + channel_to = random.choice(nonzero) + if channel_from == channel_to: + return config + proposed_balance_from = config[channel_from] - redistribution_amount + proposed_balance_to = config[channel_to] + redistribution_amount + if balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): + return config + else: + config[channel_from] = proposed_balance_from + config[channel_to] = proposed_balance_to + assert sum([cv for cv in config.values()]) == amount_msat + return config + + def split(config: dict): + # we split off a certain amount from a nonzero channel and put it into a + # zero channel + nonzero = [ck for ck, cv in config.items() if cv != 0] + zero = [ck for ck, cv in config.items() if cv == 0] + try: + channel_from = random.choice(nonzero) + channel_to = random.choice(zero) + except IndexError: + return config + delta = config[channel_from] // SPLIT_FRACTION + proposed_balance_from = config[channel_from] - delta + proposed_balance_to = config[channel_to] + delta + if balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): + return config + else: + config[channel_from] = proposed_balance_from + config[channel_to] = proposed_balance_to + assert sum([cv for cv in config.values()]) == amount_msat + return config + + def swap(config: dict): + # we swap the amounts from a single channel with another channel + nonzero = [ck for ck, cv in config.items() if cv != 0] + all = list(config.keys()) + + channel_from = random.choice(nonzero) + channel_to = random.choice(all) + + proposed_balance_to = config[channel_from] + proposed_balance_from = config[channel_to] + if balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): + return config + else: + config[channel_to] = proposed_balance_to + config[channel_from] = proposed_balance_from + return config + + initial_number_parts = number_nonzero_parts(configuration) + + for _ in range(REDISTRIBUTE): + configuration = redistribute(configuration) + if not preserve_number_parts and number_nonzero_parts( + configuration) == initial_number_parts: + configuration = split(configuration) + configuration = swap(configuration) + + return configuration + + +@profiler +def suggest_splits(amount_msat: int, channels_with_funds, exclude_single_parts=True) -> Sequence[Tuple[Dict[bytes, int], float]]: + """Creates split configurations for a payment over channels. Single channel + payments are excluded by default.""" + def rate_configuration(config: dict) -> float: + """Defines an objective function to rate a split configuration. + + We calculate the normalized L2 norm for a split configuration and + add a part penalty for each nonzero amount. The consequence is that + amounts that are equally distributed and have less parts are rated + lowest.""" + F = 0 + amount = sum([v for v in config.values()]) + + for channel, value in config.items(): + if value: + value /= amount # normalize + F += value * value + PART_PENALTY * PART_PENALTY + return F + + def rated_sorted_configurations(hierarchy: dict) -> Sequence[Tuple[Dict[bytes, int], float]]: + """Cleans up duplicate splittings, rates and sorts them according to + the rating. A lower rating is a better configuration.""" + hierarchy = unique_hierarchy(hierarchy) + rated_configs = [] + for level, configs in hierarchy.items(): + for config in configs: + rated_configs.append((config, rate_configuration(config))) + sorted_rated_configs = sorted(rated_configs, key=lambda c: c[1], reverse=False) + return sorted_rated_configs + + # create initial guesses + split_hierarchy = create_starting_split_hierarchy(amount_msat, channels_with_funds) + + # randomize initial guesses + MAX_PARTS = 5 + # generate splittings of different split levels up to number of channels + for level in range(2, min(MAX_PARTS, len(channels_with_funds) + 1)): + # generate a set of random configurations for each level + for _ in range(CANDIDATES_PER_LEVEL): + configurations = unique_hierarchy(split_hierarchy).get(level, None) + if configurations: # we have a splitting of the desired number of parts + configuration = random.choice(configurations) + # generate new splittings preserving the number of parts + configuration = propose_new_configuration( + channels_with_funds, configuration, amount_msat, + preserve_number_parts=True) + else: + # go one level lower and look for valid splittings, + # try to go one level higher by splitting a single outgoing amount + configurations = unique_hierarchy(split_hierarchy).get(level - 1, None) + if not configurations: + continue + configuration = random.choice(configurations) + # generate new splittings going one level higher in the number of parts + configuration = propose_new_configuration( + channels_with_funds, configuration, amount_msat, + preserve_number_parts=False) + + # add the newly found configuration (doesn't matter if nothing changed) + split_hierarchy[number_nonzero_parts(configuration)].append(configuration) + + if exclude_single_parts: + # we only want to return configurations that have at least two parts + try: + del split_hierarchy[1] + except: + pass + + return rated_sorted_configurations(split_hierarchy) diff --git a/electrum/tests/test_mpp_split.py b/electrum/tests/test_mpp_split.py new file mode 100644 index 000000000..b7bad140a --- /dev/null +++ b/electrum/tests/test_mpp_split.py @@ -0,0 +1,75 @@ +import random + +import electrum.mpp_split as mpp_split # side effect for PART_PENALTY +from electrum.lnutil import NoPathFound + +from . import ElectrumTestCase + +PART_PENALTY = mpp_split.PART_PENALTY + + +class TestMppSplit(ElectrumTestCase): + def setUp(self): + super().setUp() + random.seed(0) # split should only weakly depend on the seed + # test is dependent on the python version used, here 3.8 + # undo side effect + mpp_split.PART_PENALTY = PART_PENALTY + self.channels_with_funds = { + 0: 1_000_000_000, + 1: 500_000_000, + 2: 302_000_000, + 3: 101_000_000, + } + + def test_suggest_splits(self): + with self.subTest(msg="do a payment with the maximal amount spendable over a single channel"): + splits = mpp_split.suggest_splits(1_000_000_000, self.channels_with_funds, exclude_single_parts=True) + self.assertEqual({0: 500_000_000, 1: 500_000_000, 2: 0, 3: 0}, splits[0][0]) + + with self.subTest(msg="do a payment with a larger amount than what is supported by a single channel"): + splits = mpp_split.suggest_splits(1_100_000_000, self.channels_with_funds, exclude_single_parts=True) + self.assertEqual({0: 798_000_000, 1: 0, 2: 302_000_000, 3: 0}, splits[0][0]) + self.assertEqual({0: 908_000_000, 1: 0, 2: 192_000_000, 3: 0}, splits[1][0]) + + with self.subTest(msg="do a payment with the maximal amount spendable over all channels"): + splits = mpp_split.suggest_splits(sum(self.channels_with_funds.values()), self.channels_with_funds, exclude_single_parts=True) + self.assertEqual({0: 1_000_000_000, 1: 500_000_000, 2: 302_000_000, 3: 101_000_000}, splits[0][0]) + + with self.subTest(msg="do a payment with the amount supported by all channels"): + splits = mpp_split.suggest_splits(101_000_000, self.channels_with_funds, exclude_single_parts=False) + for s in splits[:4]: + self.assertEqual(1, mpp_split.number_nonzero_parts(s[0])) + + def test_payment_below_min_part_size(self): + amount = mpp_split.MIN_PART_MSAT // 2 + splits = mpp_split.suggest_splits(amount, self.channels_with_funds, exclude_single_parts=False) + # we only get four configurations that end up spending the full amount + # in a single channel + self.assertEqual(4, len(splits)) + + def test_suggest_part_penalty(self): + """Test is mainly for documentation purposes. + Decreasing the part penalty from 1.0 towards 0.0 leads to an increase + in the number of parts a payment is split. A configuration which has + about equally distributed amounts will result.""" + with self.subTest(msg="split payments with intermediate part penalty"): + mpp_split.PART_PENALTY = 0.3 + splits = mpp_split.suggest_splits(1_100_000_000, self.channels_with_funds) + self.assertEqual({0: 408_000_000, 1: 390_000_000, 2: 302_000_000, 3: 0}, splits[0][0]) + + with self.subTest(msg="split payments with no part penalty"): + mpp_split.PART_PENALTY = 0.0 + splits = mpp_split.suggest_splits(1_100_000_000, self.channels_with_funds) + self.assertEqual({0: 307_000_000, 1: 390_000_000, 2: 302_000_000, 3: 101_000_000}, splits[0][0]) + + def test_suggest_splits_single_channel(self): + channels_with_funds = { + 0: 1_000_000_000, + } + + with self.subTest(msg="do a payment with the maximal amount spendable on a single channel"): + splits = mpp_split.suggest_splits(1_000_000_000, channels_with_funds, exclude_single_parts=False) + self.assertEqual({0: 1_000_000_000}, splits[0][0]) + with self.subTest(msg="test sending an amount greater than what we have available"): + self.assertRaises(NoPathFound, mpp_split.suggest_splits, *(1_100_000_000, channels_with_funds))