Browse Source
- The splitting algorithm is redesigned to use random distribution of subsplittings over channels. - Splittings can include multiple subamounts within a channel. - The single-channel splittings are implicitly activated once the liquidity hints don't support payments of large size.master
bitromortac
5 years ago
3 changed files with 217 additions and 276 deletions
@ -1,259 +1,181 @@ |
|||||||
import random |
import random |
||||||
import math |
import math |
||||||
from typing import List, Tuple, Optional, Sequence, Dict, TYPE_CHECKING |
from typing import List, Tuple, Dict, NamedTuple |
||||||
from collections import defaultdict |
from collections import defaultdict |
||||||
|
|
||||||
from .util import profiler |
|
||||||
from .lnutil import NoPathFound |
from .lnutil import NoPathFound |
||||||
|
|
||||||
PART_PENALTY = 1.0 # 1.0 results in avoiding splits |
PART_PENALTY = 1.0 # 1.0 results in avoiding splits |
||||||
MIN_PART_MSAT = 10_000_000 # we don't want to split indefinitely |
MIN_PART_SIZE_MSAT = 10_000_000 # we don't want to split indefinitely |
||||||
EXHAUST_DECAY_FRACTION = 10 # fraction of the local balance that should be reserved if possible |
EXHAUST_DECAY_FRACTION = 10 # fraction of the local balance that should be reserved if possible |
||||||
|
RELATIVE_SPLIT_SPREAD = 0.3 # deviation from the mean when splitting amounts into parts |
||||||
# these parameters determine the granularity of the newly suggested configurations |
|
||||||
REDISTRIBUTION_FRACTION = 50 |
|
||||||
SPLIT_FRACTION = 50 |
|
||||||
|
|
||||||
# these parameters affect the computational work in the probabilistic algorithm |
# these parameters affect the computational work in the probabilistic algorithm |
||||||
STARTING_CONFIGS = 50 |
CANDIDATES_PER_LEVEL = 20 |
||||||
CANDIDATES_PER_LEVEL = 10 |
MAX_PARTS = 5 # maximum number of parts for splitting |
||||||
REDISTRIBUTE = 20 |
|
||||||
|
|
||||||
# maximum number of parts for splitting |
# maps a channel (channel_id, node_id) to a list of amounts |
||||||
MAX_PARTS = 5 |
SplitConfig = Dict[Tuple[bytes, bytes], List[int]] |
||||||
|
# maps a channel (channel_id, node_id) to the funds it has available |
||||||
|
ChannelsFundsInfo = Dict[Tuple[bytes, bytes], int] |
||||||
def unique_hierarchy(hierarchy: Dict[int, List[Dict[Tuple[bytes, bytes], int]]]) -> Dict[int, List[Dict[Tuple[bytes, bytes], int]]]: |
|
||||||
new_hierarchy = defaultdict(list) |
|
||||||
for number_parts, configs in hierarchy.items(): |
class SplitConfigRating(NamedTuple): |
||||||
unique_configs = set() |
config: SplitConfig |
||||||
for config in configs: |
rating: float |
||||||
# config dict can be out of order, so sort, otherwise not unique |
|
||||||
unique_configs.add(tuple((c, config[c]) for c in sorted(config.keys()))) |
|
||||||
for unique_config in sorted(unique_configs): |
def split_amount_normal(total_amount: int, num_parts: int) -> List[int]: |
||||||
new_hierarchy[number_parts].append( |
"""Splits an amount into about `num_parts` parts, where the parts are split |
||||||
{t[0]: t[1] for t in unique_config}) |
randomly (normally distributed around amount/num_parts with certain spread).""" |
||||||
return new_hierarchy |
parts = [] |
||||||
|
avg_amount = total_amount / num_parts |
||||||
|
# roughly reach total_amount |
||||||
def single_node_hierarchy(hierarchy: Dict[int, List[Dict[Tuple[bytes, bytes], int]]]) -> Dict[int, List[Dict[Tuple[bytes, bytes], int]]]: |
while total_amount - sum(parts) > avg_amount: |
||||||
new_hierarchy = defaultdict(list) |
amount_to_add = int(abs(random.gauss(avg_amount, RELATIVE_SPLIT_SPREAD * avg_amount))) |
||||||
for number_parts, configs in hierarchy.items(): |
if sum(parts) + amount_to_add < total_amount: |
||||||
for config in configs: |
parts.append(amount_to_add) |
||||||
# determine number of nodes in configuration |
# add what's missing |
||||||
if number_nonzero_nodes(config) > 1: |
parts.append(total_amount - sum(parts)) |
||||||
continue |
return parts |
||||||
new_hierarchy[number_parts].append(config) |
|
||||||
return new_hierarchy |
|
||||||
|
def number_parts(config: SplitConfig) -> int: |
||||||
|
return sum([len(v) for v in config.values() if sum(v)]) |
||||||
|
|
||||||
|
|
||||||
|
def number_nonzero_channels(config: SplitConfig) -> int: |
||||||
|
return len([v for v in config.values() if sum(v)]) |
||||||
|
|
||||||
|
|
||||||
|
def number_nonzero_nodes(config: SplitConfig) -> int: |
||||||
|
# using a set comprehension |
||||||
|
return len({nodeid for (_, nodeid), amounts in config.items() if sum(amounts)}) |
||||||
|
|
||||||
|
|
||||||
|
def total_config_amount(config: SplitConfig) -> int: |
||||||
|
return sum([sum(c) for c in config.values()]) |
||||||
|
|
||||||
|
|
||||||
def number_nonzero_parts(configuration: Dict[Tuple[bytes, bytes], int]) -> int: |
def is_any_amount_smaller_than_min_part_size(config: SplitConfig) -> bool: |
||||||
return len([v for v in configuration.values() if v]) |
smaller = False |
||||||
|
for amounts in config.values(): |
||||||
|
if any([amount < MIN_PART_SIZE_MSAT for amount in amounts]): |
||||||
|
smaller |= True |
||||||
|
return smaller |
||||||
|
|
||||||
|
|
||||||
def number_nonzero_nodes(configuration: Dict[Tuple[bytes, bytes], int]) -> int: |
def remove_duplicates(configs: List[SplitConfig]) -> List[SplitConfig]: |
||||||
return len({nodeid for (_, nodeid), amount in configuration.items() if amount > 0}) |
unique_configs = set() |
||||||
|
for config in configs: |
||||||
|
# sort keys and values |
||||||
|
config_sorted_values = {k: sorted(v) for k, v in config.items()} |
||||||
|
config_sorted_keys = {k: config_sorted_values[k] for k in sorted(config_sorted_values.keys())} |
||||||
|
hashable_config = tuple((c, tuple(sorted(config[c]))) for c in config_sorted_keys) |
||||||
|
unique_configs.add(hashable_config) |
||||||
|
unique_configs = [{c[0]: list(c[1]) for c in config} for config in unique_configs] |
||||||
|
return unique_configs |
||||||
|
|
||||||
|
|
||||||
def create_starting_split_hierarchy(amount_msat: int, channels_with_funds: Dict[Tuple[bytes, bytes], int]): |
def remove_multiple_nodes(configs: List[SplitConfig]) -> List[SplitConfig]: |
||||||
"""Distributes the amount to send to a single or more channels in several |
return [config for config in configs if number_nonzero_nodes(config) == 1] |
||||||
ways (randomly).""" |
|
||||||
# TODO: find all possible starting configurations deterministically |
|
||||||
# could try all permutations |
|
||||||
|
|
||||||
split_hierarchy = defaultdict(list) |
|
||||||
|
def remove_single_part_configs(configs: List[SplitConfig]) -> List[SplitConfig]: |
||||||
|
return [config for config in configs if number_parts(config) != 1] |
||||||
|
|
||||||
|
|
||||||
|
def rate_config( |
||||||
|
config: SplitConfig, |
||||||
|
channels_with_funds: ChannelsFundsInfo) -> float: |
||||||
|
"""Defines an objective function to rate a configuration. |
||||||
|
|
||||||
|
We calculate the normalized L2 norm for a configuration and |
||||||
|
add a part penalty for each nonzero amount. The consequence is that |
||||||
|
amounts that are equally distributed and have less parts are rated |
||||||
|
lowest (best). A penalty depending on the total amount sent over a channel |
||||||
|
counteracts channel exhaustion.""" |
||||||
|
rating = 0 |
||||||
|
total_amount = total_config_amount(config) |
||||||
|
|
||||||
|
for channel, amounts in config.items(): |
||||||
|
funds = channels_with_funds[channel] |
||||||
|
if amounts: |
||||||
|
for amount in amounts: |
||||||
|
rating += amount * amount / (total_amount * total_amount) # penalty to favor equal distribution of amounts |
||||||
|
rating += PART_PENALTY * PART_PENALTY # penalty for each part |
||||||
|
decay = funds / EXHAUST_DECAY_FRACTION |
||||||
|
rating += math.exp((sum(amounts) - funds) / decay) # penalty for channel exhaustion |
||||||
|
return rating |
||||||
|
|
||||||
|
|
||||||
|
def suggest_splits( |
||||||
|
amount_msat: int, channels_with_funds: ChannelsFundsInfo, |
||||||
|
exclude_single_part_payments=False, |
||||||
|
exclude_multinode_payments=False |
||||||
|
) -> List[SplitConfigRating]: |
||||||
|
"""Breaks amount_msat into smaller pieces and distributes them over the |
||||||
|
channels according to the funds they can send. |
||||||
|
|
||||||
|
Individual channels may be assigned multiple parts. The split configurations |
||||||
|
are returned in sorted order, from best to worst rating. |
||||||
|
|
||||||
|
Single part payments can be excluded, since they represent legacy payments. |
||||||
|
Split configurations that send via multiple nodes can be excluded as well. |
||||||
|
""" |
||||||
|
|
||||||
|
configs = [] |
||||||
channels_order = list(channels_with_funds.keys()) |
channels_order = list(channels_with_funds.keys()) |
||||||
|
|
||||||
for _ in range(STARTING_CONFIGS): |
# generate multiple configurations to get more configurations (there is randomness in this loop) |
||||||
# shuffle to have different starting points |
for _ in range(CANDIDATES_PER_LEVEL): |
||||||
random.shuffle(channels_order) |
# we want to have configurations with no splitting to many splittings |
||||||
|
for target_parts in range(1, MAX_PARTS): |
||||||
configuration = {} |
config = defaultdict(list) # type: SplitConfig |
||||||
amount_added = 0 |
|
||||||
for c in channels_order: |
# randomly split amount into target_parts chunks |
||||||
s = channels_with_funds[c] |
split_amounts = split_amount_normal(amount_msat, target_parts) |
||||||
if amount_added == amount_msat: |
# randomly distribute amounts over channels |
||||||
configuration[c] = 0 |
for amount in split_amounts: |
||||||
else: |
random.shuffle(channels_order) |
||||||
amount_to_add = amount_msat - amount_added |
# we check each channel and try to put the funds inside, break if we succeed |
||||||
amt = min(s, amount_to_add) |
for c in channels_order: |
||||||
configuration[c] = amt |
if sum(config[c]) + amount <= channels_with_funds[c]: |
||||||
amount_added += amt |
config[c].append(amount) |
||||||
if amount_added != amount_msat: |
break |
||||||
raise NoPathFound("Channels don't have enough sending capacity.") |
# if we don't succeed to put the amount anywhere, |
||||||
split_hierarchy[number_nonzero_parts(configuration)].append(configuration) |
# we try to fill up channels and put the rest somewhere else |
||||||
|
else: |
||||||
return unique_hierarchy(split_hierarchy) |
distribute_amount = amount |
||||||
|
for c in channels_order: |
||||||
|
funds_left = channels_with_funds[c] - sum(config[c]) |
||||||
def balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): |
# it would be good to not fill the full channel if possible |
||||||
check = ( |
add_amount = min(funds_left, distribute_amount) |
||||||
proposed_balance_to < MIN_PART_MSAT or |
config[c].append(add_amount) |
||||||
proposed_balance_to > channels_with_funds[channel_to] or |
distribute_amount -= add_amount |
||||||
proposed_balance_from < MIN_PART_MSAT or |
if distribute_amount == 0: |
||||||
proposed_balance_from > channels_with_funds[channel_from] |
break |
||||||
) |
if total_config_amount(config) != amount_msat: |
||||||
return check |
raise NoPathFound('Cannot distribute payment over channels.') |
||||||
|
if target_parts > 1 and is_any_amount_smaller_than_min_part_size(config): |
||||||
|
continue |
||||||
def propose_new_configuration(channels_with_funds: Dict[Tuple[bytes, bytes], int], configuration: Dict[Tuple[bytes, bytes], int], |
assert total_config_amount(config) == amount_msat |
||||||
amount_msat: int, preserve_number_parts=True) -> Dict[Tuple[bytes, bytes], int]: |
configs.append(config) |
||||||
"""Randomly alters a split configuration. If preserve_number_parts, the |
|
||||||
configuration stays within the same class of number of splits.""" |
configs = remove_duplicates(configs) |
||||||
|
|
||||||
# there are three basic operations to reach different split configurations: |
# we only take configurations that send via a single node (but there can be multiple parts) |
||||||
# redistribute, split, swap |
if exclude_multinode_payments: |
||||||
|
configs = remove_multiple_nodes(configs) |
||||||
def redistribute(config: dict): |
|
||||||
# we redistribute the amount from a nonzero channel to a nonzero channel |
if exclude_single_part_payments: |
||||||
redistribution_amount = amount_msat // REDISTRIBUTION_FRACTION |
configs = remove_single_part_configs(configs) |
||||||
nonzero = [ck for ck, cv in config.items() if |
|
||||||
cv >= redistribution_amount] |
rated_configs = [SplitConfigRating( |
||||||
if len(nonzero) == 1: # we only have a single channel, so we can't redistribute |
config=c, |
||||||
return config |
rating=rate_config(c, channels_with_funds) |
||||||
|
) for c in configs] |
||||||
channel_from = random.choice(nonzero) |
rated_configs.sort(key=lambda x: x.rating) |
||||||
channel_to = random.choice(nonzero) |
|
||||||
if channel_from == channel_to: |
return rated_configs |
||||||
return config |
|
||||||
proposed_balance_from = config[channel_from] - redistribution_amount |
|
||||||
proposed_balance_to = config[channel_to] + redistribution_amount |
|
||||||
if balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): |
|
||||||
return config |
|
||||||
else: |
|
||||||
config[channel_from] = proposed_balance_from |
|
||||||
config[channel_to] = proposed_balance_to |
|
||||||
assert sum([cv for cv in config.values()]) == amount_msat |
|
||||||
return config |
|
||||||
|
|
||||||
def split(config: dict): |
|
||||||
# we split off a certain amount from a nonzero channel and put it into a |
|
||||||
# zero channel |
|
||||||
nonzero = [ck for ck, cv in config.items() if cv != 0] |
|
||||||
zero = [ck for ck, cv in config.items() if cv == 0] |
|
||||||
try: |
|
||||||
channel_from = random.choice(nonzero) |
|
||||||
channel_to = random.choice(zero) |
|
||||||
except IndexError: |
|
||||||
return config |
|
||||||
delta = config[channel_from] // SPLIT_FRACTION |
|
||||||
proposed_balance_from = config[channel_from] - delta |
|
||||||
proposed_balance_to = config[channel_to] + delta |
|
||||||
if balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): |
|
||||||
return config |
|
||||||
else: |
|
||||||
config[channel_from] = proposed_balance_from |
|
||||||
config[channel_to] = proposed_balance_to |
|
||||||
assert sum([cv for cv in config.values()]) == amount_msat |
|
||||||
return config |
|
||||||
|
|
||||||
def swap(config: dict): |
|
||||||
# we swap the amounts from a single channel with another channel |
|
||||||
nonzero = [ck for ck, cv in config.items() if cv != 0] |
|
||||||
all = list(config.keys()) |
|
||||||
|
|
||||||
channel_from = random.choice(nonzero) |
|
||||||
channel_to = random.choice(all) |
|
||||||
|
|
||||||
proposed_balance_to = config[channel_from] |
|
||||||
proposed_balance_from = config[channel_to] |
|
||||||
if balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): |
|
||||||
return config |
|
||||||
else: |
|
||||||
config[channel_to] = proposed_balance_to |
|
||||||
config[channel_from] = proposed_balance_from |
|
||||||
return config |
|
||||||
|
|
||||||
initial_number_parts = number_nonzero_parts(configuration) |
|
||||||
|
|
||||||
for _ in range(REDISTRIBUTE): |
|
||||||
configuration = redistribute(configuration) |
|
||||||
if not preserve_number_parts and number_nonzero_parts( |
|
||||||
configuration) == initial_number_parts: |
|
||||||
configuration = split(configuration) |
|
||||||
configuration = swap(configuration) |
|
||||||
|
|
||||||
return configuration |
|
||||||
|
|
||||||
|
|
||||||
@profiler |
|
||||||
def suggest_splits(amount_msat: int, channels_with_funds: Dict[Tuple[bytes, bytes], int], |
|
||||||
exclude_single_parts=True, single_node=False) \ |
|
||||||
-> Sequence[Tuple[Dict[Tuple[bytes, bytes], int], float]]: |
|
||||||
"""Creates split configurations for a payment over channels. Single channel |
|
||||||
payments are excluded by default. channels_with_funds is keyed by |
|
||||||
(channelid, nodeid).""" |
|
||||||
|
|
||||||
def rate_configuration(config: dict) -> float: |
|
||||||
"""Defines an objective function to rate a split configuration. |
|
||||||
|
|
||||||
We calculate the normalized L2 norm for a split configuration and |
|
||||||
add a part penalty for each nonzero amount. The consequence is that |
|
||||||
amounts that are equally distributed and have less parts are rated |
|
||||||
lowest.""" |
|
||||||
F = 0 |
|
||||||
total_amount = sum([v for v in config.values()]) |
|
||||||
|
|
||||||
for channel, amount in config.items(): |
|
||||||
funds = channels_with_funds[channel] |
|
||||||
if amount: |
|
||||||
F += amount * amount / (total_amount * total_amount) # a penalty to favor distribution of amounts |
|
||||||
F += PART_PENALTY * PART_PENALTY # a penalty for each part |
|
||||||
decay = funds / EXHAUST_DECAY_FRACTION |
|
||||||
F += math.exp((amount - funds) / decay) # a penalty for channel saturation |
|
||||||
|
|
||||||
return F |
|
||||||
|
|
||||||
def rated_sorted_configurations(hierarchy: dict) -> Sequence[Tuple[Dict[Tuple[bytes, bytes], int], float]]: |
|
||||||
"""Cleans up duplicate splittings, rates and sorts them according to |
|
||||||
the rating. A lower rating is a better configuration.""" |
|
||||||
hierarchy = unique_hierarchy(hierarchy) |
|
||||||
rated_configs = [] |
|
||||||
for level, configs in hierarchy.items(): |
|
||||||
for config in configs: |
|
||||||
rated_configs.append((config, rate_configuration(config))) |
|
||||||
sorted_rated_configs = sorted(rated_configs, key=lambda c: c[1], reverse=False) |
|
||||||
return sorted_rated_configs |
|
||||||
|
|
||||||
# create initial guesses |
|
||||||
split_hierarchy = create_starting_split_hierarchy(amount_msat, channels_with_funds) |
|
||||||
|
|
||||||
# randomize initial guesses and generate splittings of different split |
|
||||||
# levels up to number of channels |
|
||||||
for level in range(2, min(MAX_PARTS, len(channels_with_funds) + 1)): |
|
||||||
# generate a set of random configurations for each level |
|
||||||
for _ in range(CANDIDATES_PER_LEVEL): |
|
||||||
configurations = unique_hierarchy(split_hierarchy).get(level, None) |
|
||||||
if configurations: # we have a splitting of the desired number of parts |
|
||||||
configuration = random.choice(configurations) |
|
||||||
# generate new splittings preserving the number of parts |
|
||||||
configuration = propose_new_configuration( |
|
||||||
channels_with_funds, configuration, amount_msat, |
|
||||||
preserve_number_parts=True) |
|
||||||
else: |
|
||||||
# go one level lower and look for valid splittings, |
|
||||||
# try to go one level higher by splitting a single outgoing amount |
|
||||||
configurations = unique_hierarchy(split_hierarchy).get(level - 1, None) |
|
||||||
if not configurations: |
|
||||||
continue |
|
||||||
configuration = random.choice(configurations) |
|
||||||
# generate new splittings going one level higher in the number of parts |
|
||||||
configuration = propose_new_configuration( |
|
||||||
channels_with_funds, configuration, amount_msat, |
|
||||||
preserve_number_parts=False) |
|
||||||
|
|
||||||
# add the newly found configuration (doesn't matter if nothing changed) |
|
||||||
split_hierarchy[number_nonzero_parts(configuration)].append(configuration) |
|
||||||
|
|
||||||
if exclude_single_parts: |
|
||||||
# we only want to return configurations that have at least two parts |
|
||||||
try: |
|
||||||
del split_hierarchy[1] |
|
||||||
except: |
|
||||||
pass |
|
||||||
|
|
||||||
if single_node: |
|
||||||
# we only take configurations that send to a single node |
|
||||||
split_hierarchy = single_node_hierarchy(split_hierarchy) |
|
||||||
|
|
||||||
return rated_sorted_configurations(split_hierarchy) |
|
||||||
|
|||||||
Loading…
Reference in new issue