@ -23,6 +23,8 @@
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
import datetime
import random
import queue import queue
import os import os
import json import json
@ -33,6 +35,14 @@ import binascii
import base64 import base64
import asyncio import asyncio
from sqlalchemy import create_engine , Column , ForeignKey , Integer , String , DateTime , Boolean
from sqlalchemy . engine import Engine
from sqlalchemy . orm import sessionmaker
from sqlalchemy . orm . query import Query
from sqlalchemy . ext . declarative import declarative_base
from sqlalchemy . sql import not_ , or_
from sqlalchemy . orm import scoped_session
from . import constants from . import constants
from . util import PrintError , bh2u , profiler , get_headers_dir , bfh , is_ip_address , list_enabled_bits from . util import PrintError , bh2u , profiler , get_headers_dir , bfh , is_ip_address , list_enabled_bits
from . storage import JsonDB from . storage import JsonDB
@ -41,112 +51,113 @@ from .crypto import sha256d
from . import ecc from . import ecc
from . lnutil import ( LN_GLOBAL_FEATURES_KNOWN_SET , LNPeerAddr , NUM_MAX_EDGES_IN_PAYMENT_PATH , from . lnutil import ( LN_GLOBAL_FEATURES_KNOWN_SET , LNPeerAddr , NUM_MAX_EDGES_IN_PAYMENT_PATH ,
NotFoundChanAnnouncementForUpdate ) NotFoundChanAnnouncementForUpdate )
from . lnmsg import encode_msg
if TYPE_CHECKING : if TYPE_CHECKING :
from . lnchannel import Channel from . lnchannel import Channel
from . network import Network from . network import Network
class UnknownEvenFeatureBits ( Exception ) : pass class UnknownEvenFeatureBits ( Exception ) : pass
class NoChannelPolicy ( Exception ) :
def __init__ ( self , short_channel_id : bytes ) :
super ( ) . __init__ ( f ' cannot find channel policy for short_channel_id: { bh2u ( short_channel_id ) } ' )
class ChannelInfo ( PrintError ) :
def validate_features ( features : int ) :
def __init__ ( self , channel_announcement_payload ) : enabled_features = list_enabled_bits ( features )
self . features_len = channel_announcement_payload [ ' len ' ] for fbit in enabled_features :
self . features = channel_announcement_payload [ ' features ' ] if ( 1 << fbit ) not in LN_GLOBAL_FEATURES_KNOWN_SET and fbit % 2 == 0 :
enabled_features = list_enabled_bits ( int . from_bytes ( self . features , " big " ) ) raise UnknownEvenFeatureBits ( )
for fbit in enabled_features :
if ( 1 << fbit ) not in LN_GLOBAL_FEATURES_KNOWN_SET and fbit % 2 == 0 : Base = declarative_base ( )
raise UnknownEvenFeatureBits ( ) session_factory = sessionmaker ( )
DBSession = scoped_session ( session_factory )
self . channel_id = channel_announcement_payload [ ' short_channel_id ' ] engine = None
self . node_id_1 = channel_announcement_payload [ ' node_id_1 ' ]
self . node_id_2 = channel_announcement_payload [ ' node_id_2 ' ] FLAG_DISABLE = 1 << 1
assert type ( self . node_id_1 ) is bytes FLAG_DIRECTION = 1 << 0
assert type ( self . node_id_2 ) is bytes
assert list ( sorted ( [ self . node_id_1 , self . node_id_2 ] ) ) == [ self . node_id_1 , self . node_id_2 ] class ChannelInfoInDB ( Base ) :
__tablename__ = ' channel_info '
self . bitcoin_key_1 = channel_announcement_payload [ ' bitcoin_key_1 ' ] short_channel_id = Column ( String ( 64 ) , primary_key = True )
self . bitcoin_key_2 = channel_announcement_payload [ ' bitcoin_key_2 ' ] node1_id = Column ( String ( 66 ) , ForeignKey ( ' node_info.node_id ' ) , nullable = False )
node2_id = Column ( String ( 66 ) , ForeignKey ( ' node_info.node_id ' ) , nullable = False )
# this field does not get persisted capacity_sat = Column ( Integer )
self . msg_payload = channel_announcement_payload msg_payload_hex = Column ( String ( 1024 ) , nullable = False )
trusted = Column ( Boolean , nullable = False )
self . capacity_sat = None
self . policy_node1 = None @staticmethod
self . policy_node2 = None def from_msg ( channel_announcement_payload ) :
features = int . from_bytes ( channel_announcement_payload [ ' features ' ] , ' big ' )
def to_json ( self ) - > dict : validate_features ( features )
d = { }
d [ ' short_channel_id ' ] = bh2u ( self . channel_id ) channel_id = channel_announcement_payload [ ' short_channel_id ' ] . hex ( )
d [ ' node_id_1 ' ] = bh2u ( self . node_id_1 ) node_id_1 = channel_announcement_payload [ ' node_id_1 ' ] . hex ( )
d [ ' node_id_2 ' ] = bh2u ( self . node_id_2 ) node_id_2 = channel_announcement_payload [ ' node_id_2 ' ] . hex ( )
d [ ' len ' ] = bh2u ( self . features_len ) assert list ( sorted ( [ node_id_1 , node_id_2 ] ) ) == [ node_id_1 , node_id_2 ]
d [ ' features ' ] = bh2u ( self . features )
d [ ' bitcoin_key_1 ' ] = bh2u ( self . bitcoin_key_1 ) msg_payload_hex = encode_msg ( ' channel_announcement ' , * * channel_announcement_payload ) . hex ( )
d [ ' bitcoin_key_2 ' ] = bh2u ( self . bitcoin_key_2 )
d [ ' policy_node1 ' ] = self . policy_node1 capacity_sat = None
d [ ' policy_node2 ' ] = self . policy_node2
d [ ' capacity_sat ' ] = self . capacity_sat return ChannelInfoInDB ( short_channel_id = channel_id , node1_id = node_id_1 ,
return d node2_id = node_id_2 , capacity_sat = capacity_sat , msg_payload_hex = msg_payload_hex ,
trusted = False )
@classmethod
def from_json ( cls , d : dict ) : @property
d2 = { } def msg_payload ( self ) :
d2 [ ' short_channel_id ' ] = bfh ( d [ ' short_channel_id ' ] ) return bytes . fromhex ( self . msg_payload_hex )
d2 [ ' node_id_1 ' ] = bfh ( d [ ' node_id_1 ' ] )
d2 [ ' node_id_2 ' ] = bfh ( d [ ' node_id_2 ' ] ) def on_channel_update ( self , msg : dict , trusted = False ) :
d2 [ ' len ' ] = bfh ( d [ ' len ' ] ) assert self . short_channel_id == msg [ ' short_channel_id ' ] . hex ( )
d2 [ ' features ' ] = bfh ( d [ ' features ' ] ) flags = int . from_bytes ( msg [ ' channel_flags ' ] , ' big ' )
d2 [ ' bitcoin_key_1 ' ] = bfh ( d [ ' bitcoin_key_1 ' ] ) direction = flags & FLAG_DIRECTION
d2 [ ' bitcoin_key_2 ' ] = bfh ( d [ ' bitcoin_key_2 ' ] )
ci = ChannelInfo ( d2 )
ci . capacity_sat = d [ ' capacity_sat ' ]
ci . policy_node1 = ChannelInfoDirectedPolicy . from_json ( d [ ' policy_node1 ' ] )
ci . policy_node2 = ChannelInfoDirectedPolicy . from_json ( d [ ' policy_node2 ' ] )
return ci
def set_capacity ( self , capacity ) :
self . capacity_sat = capacity
def on_channel_update ( self , msg_payload , trusted = False ) :
assert self . channel_id == msg_payload [ ' short_channel_id ' ]
flags = int . from_bytes ( msg_payload [ ' channel_flags ' ] , ' big ' )
direction = flags & ChannelInfoDirectedPolicy . FLAG_DIRECTION
new_policy = ChannelInfoDirectedPolicy ( msg_payload )
if direction == 0 : if direction == 0 :
old_policy = self . policy_node1 node_id = self . node1_id
node_id = self . node_id_1
else : else :
old_policy = self . policy_node2 node_id = self . node2_id
node_id = self . node_id_2 new_policy = Policy . from_msg ( msg , node_id , self . short_channel_id )
if old_policy and old_policy . timestamp > = new_policy . timestamp : old_policy = DBSession . query ( Policy ) . filter_by ( short_channel_id = self . short_channel_id , start_node = node_id ) . one_or_none ( )
if not old_policy :
DBSession . add ( new_policy )
return
if old_policy . timestamp > = new_policy . timestamp :
return # ignore return # ignore
if not trusted and not verify_sig_for_channel_update ( msg_payload , node_id ) : if not trusted and not verify_sig_for_channel_update ( msg , bytes . fromhex ( node_id ) ) :
return # ignore return # ignore
# save new policy old_policy . cltv_expiry_delta = new_policy . cltv_expiry_delta
if direction == 0 : old_policy . htlc_minimum_msat = new_policy . htlc_minimum_msat
self . policy_node1 = new_policy old_policy . htlc_maximum_msat = new_policy . htlc_maximum_msat
else : old_policy . fee_base_msat = new_policy . fee_base_msat
self . policy_node2 = new_policy old_policy . fee_proportional_millionths = new_policy . fee_proportional_millionths
old_policy . channel_flags = new_policy . channel_flags
def get_policy_for_node ( self , node_id : bytes ) - > Optional [ ' ChannelInfoDirectedPolicy ' ] : old_policy . timestamp = new_policy . timestamp
if node_id == self . node_id_1 :
return self . policy_node1 def get_policy_for_node ( self , node ) - > Optional [ ' Policy ' ] :
elif node_id == self . node_id_2 : """
return self . policy_node2 raises when initiator / non - initiator both unequal node
else : """
raise Exception ( ' node_id {} not in channel {} ' . format ( node_id , self . channel_id ) ) if node . hex ( ) not in ( self . node1_id , self . node2_id ) :
raise Exception ( " the given node is not a party in this channel " )
n1 = DBSession . query ( Policy ) . filter_by ( short_channel_id = self . short_channel_id , start_node = self . node1_id ) . one_or_none ( )
class ChannelInfoDirectedPolicy : if n1 :
return n1
FLAG_DIRECTION = 1 << 0 n2 = DBSession . query ( Policy ) . filter_by ( short_channel_id = self . short_channel_id , start_node = self . node2_id ) . one_or_none ( )
FLAG_DISABLE = 1 << 1 return n2
def __init__ ( self , channel_update_payload ) : class Policy ( Base ) :
__tablename__ = ' policy '
start_node = Column ( String ( 66 ) , ForeignKey ( ' node_info.node_id ' ) , primary_key = True )
short_channel_id = Column ( String ( 64 ) , ForeignKey ( ' channel_info.short_channel_id ' ) , primary_key = True )
cltv_expiry_delta = Column ( Integer , nullable = False )
htlc_minimum_msat = Column ( Integer , nullable = False )
htlc_maximum_msat = Column ( Integer )
fee_base_msat = Column ( Integer , nullable = False )
fee_proportional_millionths = Column ( Integer , nullable = False )
channel_flags = Column ( Integer , nullable = False )
timestamp = Column ( DateTime , nullable = False )
@staticmethod
def from_msg ( channel_update_payload , start_node , short_channel_id ) :
cltv_expiry_delta = channel_update_payload [ ' cltv_expiry_delta ' ] cltv_expiry_delta = channel_update_payload [ ' cltv_expiry_delta ' ]
htlc_minimum_msat = channel_update_payload [ ' htlc_minimum_msat ' ] htlc_minimum_msat = channel_update_payload [ ' htlc_minimum_msat ' ]
fee_base_msat = channel_update_payload [ ' fee_base_msat ' ] fee_base_msat = channel_update_payload [ ' fee_base_msat ' ]
@ -155,61 +166,52 @@ class ChannelInfoDirectedPolicy:
timestamp = channel_update_payload [ ' timestamp ' ] timestamp = channel_update_payload [ ' timestamp ' ]
htlc_maximum_msat = channel_update_payload . get ( ' htlc_maximum_msat ' ) # optional htlc_maximum_msat = channel_update_payload . get ( ' htlc_maximum_msat ' ) # optional
self . cltv_expiry_delta = int . from_bytes ( cltv_expiry_delta , " big " ) cltv_expiry_delta = int . from_bytes ( cltv_expiry_delta , " big " )
self . htlc_minimum_msat = int . from_bytes ( htlc_minimum_msat , " big " ) htlc_minimum_msat = int . from_bytes ( htlc_minimum_msat , " big " )
self . htlc_maximum_msat = int . from_bytes ( htlc_maximum_msat , " big " ) if htlc_maximum_msat else None htlc_maximum_msat = int . from_bytes ( htlc_maximum_msat , " big " ) if htlc_maximum_msat else None
self . fee_base_msat = int . from_bytes ( fee_base_msat , " big " ) fee_base_msat = int . from_bytes ( fee_base_msat , " big " )
self . fee_proportional_millionths = int . from_bytes ( fee_proportional_millionths , " big " ) fee_proportional_millionths = int . from_bytes ( fee_proportional_millionths , " big " )
self . channel_flags = int . from_bytes ( channel_flags , " big " ) channel_flags = int . from_bytes ( channel_flags , " big " )
self . timestamp = int . from_bytes ( timestamp , " big " ) timestamp = datetime . datetime . fromtimestamp ( int . from_bytes ( timestamp , " big " ) )
self . disabled = self . channel_flags & self . FLAG_DISABLE return Policy ( start_node = start_node ,
short_channel_id = short_channel_id ,
def to_json ( self ) - > dict : cltv_expiry_delta = cltv_expiry_delta ,
d = { } htlc_minimum_msat = htlc_minimum_msat ,
d [ ' cltv_expiry_delta ' ] = self . cltv_expiry_delta fee_base_msat = fee_base_msat ,
d [ ' htlc_minimum_msat ' ] = self . htlc_minimum_msat fee_proportional_millionths = fee_proportional_millionths ,
d [ ' fee_base_msat ' ] = self . fee_base_msat channel_flags = channel_flags ,
d [ ' fee_proportional_millionths ' ] = self . fee_proportional_millionths timestamp = timestamp ,
d [ ' channel_flags ' ] = self . channel_flags htlc_maximum_msat = htlc_maximum_msat )
d [ ' timestamp ' ] = self . timestamp
if self . htlc_maximum_msat : def is_disabled ( self ) :
d [ ' htlc_maximum_msat ' ] = self . htlc_maximum_msat return self . channel_flags & FLAG_DISABLE
return d
class NodeInfoInDB ( Base ) :
@classmethod __tablename__ = ' node_info '
def from_json ( cls , d : dict ) : node_id = Column ( String ( 66 ) , primary_key = True , sqlite_on_conflict_primary_key = ' REPLACE ' )
if d is None : return None features = Column ( Integer , nullable = False )
d2 = { } timestamp = Column ( Integer , nullable = False )
d2 [ ' cltv_expiry_delta ' ] = d [ ' cltv_expiry_delta ' ] . to_bytes ( 2 , " big " ) alias = Column ( String ( 64 ) , nullable = False )
d2 [ ' htlc_minimum_msat ' ] = d [ ' htlc_minimum_msat ' ] . to_bytes ( 8 , " big " )
d2 [ ' htlc_maximum_msat ' ] = d [ ' htlc_maximum_msat ' ] . to_bytes ( 8 , " big " ) if d . get ( ' htlc_maximum_msat ' ) else None def get_addresses ( self ) :
d2 [ ' fee_base_msat ' ] = d [ ' fee_base_msat ' ] . to_bytes ( 4 , " big " ) return DBSession . query ( AddressInDB ) . join ( NodeInfoInDB ) . filter_by ( node_id = self . node_id ) . all ( )
d2 [ ' fee_proportional_millionths ' ] = d [ ' fee_proportional_millionths ' ] . to_bytes ( 4 , " big " )
d2 [ ' channel_flags ' ] = d [ ' channel_flags ' ] . to_bytes ( 1 , " big " ) @staticmethod
d2 [ ' timestamp ' ] = d [ ' timestamp ' ] . to_bytes ( 4 , " big " ) def from_msg ( node_announcement_payload , addresses_already_parsed = False ) :
return ChannelInfoDirectedPolicy ( d2 ) node_id = node_announcement_payload [ ' node_id ' ] . hex ( )
features = int . from_bytes ( node_announcement_payload [ ' features ' ] , " big " )
validate_features ( features )
class NodeInfo ( PrintError ) :
def __init__ ( self , node_announcement_payload , addresses_already_parsed = False ) :
self . pubkey = node_announcement_payload [ ' node_id ' ]
self . features_len = node_announcement_payload [ ' flen ' ]
self . features = node_announcement_payload [ ' features ' ]
enabled_features = list_enabled_bits ( int . from_bytes ( self . features , " big " ) )
for fbit in enabled_features :
if ( 1 << fbit ) not in LN_GLOBAL_FEATURES_KNOWN_SET and fbit % 2 == 0 :
raise UnknownEvenFeatureBits ( )
if not addresses_already_parsed : if not addresses_already_parsed :
self . addresses = self . parse_addresses_field ( node_announcement_payload [ ' addresses ' ] ) addresses = NodeInfoInDB . parse_addresses_field ( node_announcement_payload [ ' addresses ' ] )
else : else :
self . addresses = node_announcement_payload [ ' addresses ' ] addresses = node_announcement_payload [ ' addresses ' ]
self . alias = node_announcement_payload [ ' alias ' ] . rstrip ( b ' \x00 ' ) alias = node_announcement_payload [ ' alias ' ] . rstrip ( b ' \x00 ' ) . hex ( )
self . timestamp = int . from_bytes ( node_announcement_payload [ ' timestamp ' ] , " big " ) timestamp = datetime . datetime . fromtimestamp ( int . from_bytes ( node_announcement_payload [ ' timestamp ' ] , " big " ) )
return NodeInfoInDB ( node_id = node_id , features = features , timestamp = timestamp , alias = alias ) , [ AddressInDB ( host = host , port = port , node_id = node_id , last_connected_date = datetime . datetime . now ( ) ) for host , port in addresses ]
@class method @staticmethod
def parse_addresses_field ( cls , addresses_field ) : def parse_addresses_field ( addresses_field ) :
buf = addresses_field buf = addresses_field
def read ( n ) : def read ( n ) :
nonlocal buf nonlocal buf
@ -248,243 +250,233 @@ class NodeInfo(PrintError):
break break
return addresses return addresses
def to_json ( self ) - > dict : class AddressInDB ( Base ) :
d = { } __tablename__ = ' address '
d [ ' node_id ' ] = bh2u ( self . pubkey ) node_id = Column ( String ( 66 ) , ForeignKey ( ' node_info.node_id ' ) , primary_key = True )
d [ ' flen ' ] = bh2u ( self . features_len ) host = Column ( String ( 256 ) , primary_key = True )
d [ ' features ' ] = bh2u ( self . features ) port = Column ( Integer , primary_key = True )
d [ ' addresses ' ] = self . addresses last_connected_date = Column ( DateTime ( ) , nullable = False )
d [ ' alias ' ] = bh2u ( self . alias )
d [ ' timestamp ' ] = self . timestamp
return d
@classmethod
def from_json ( cls , d : dict ) :
if d is None : return None
d2 = { }
d2 [ ' node_id ' ] = bfh ( d [ ' node_id ' ] )
d2 [ ' flen ' ] = bfh ( d [ ' flen ' ] )
d2 [ ' features ' ] = bfh ( d [ ' features ' ] )
d2 [ ' addresses ' ] = d [ ' addresses ' ]
d2 [ ' alias ' ] = bfh ( d [ ' alias ' ] )
d2 [ ' timestamp ' ] = d [ ' timestamp ' ] . to_bytes ( 4 , " big " )
return NodeInfo ( d2 , addresses_already_parsed = True )
class ChannelDB ( JsonDB ) : class ChannelDB :
NUM_MAX_RECENT_PEERS = 20 NUM_MAX_RECENT_PEERS = 20
def __init__ ( self , network : ' Network ' ) : def __init__ ( self , network : ' Network ' ) :
global engine
self . network = network self . network = network
path = os . path . join ( get_headers_dir ( network . config ) , ' channel_db ' ) self . num_nodes = 0
JsonDB . __init__ ( self , path ) self . num_channels = 0
self . path = os . path . join ( get_headers_dir ( network . config ) , ' channel_db.sqlite3 ' )
engine = create_engine ( ' sqlite:/// ' + self . path ) #, echo=True)
DBSession . remove ( )
DBSession . configure ( bind = engine , autoflush = False )
Base . metadata . drop_all ( engine )
Base . metadata . create_all ( engine )
self . lock = threading . RLock ( ) self . lock = threading . RLock ( )
self . _id_to_channel_info = { } # type: Dict[bytes, ChannelInfo]
self . _channels_for_node = defaultdict ( set ) # node -> set(short_channel_id)
self . nodes = { } # node_id -> NodeInfo
self . _recent_peers = [ ]
self . _last_good_address = { } # node_id -> LNPeerAddr
# (intentionally not persisted) # (intentionally not persisted)
self . _channel_updates_for_private_channels = { } # type: Dict[Tuple[bytes, bytes], ChannelInfoDirectedPolicy ] self . _channel_updates_for_private_channels = { } # type: Dict[Tuple[bytes, bytes], dict]
self . ca_verifier = LNChannelVerifier ( network , self ) self . ca_verifier = LNChannelVerifier ( network , self )
self . load_data ( ) def update_counts ( self ) :
self . num_channels = DBSession . query ( ChannelInfoInDB ) . count ( )
def load_data ( self ) : self . num_nodes = DBSession . query ( NodeInfoInDB ) . count ( )
if os . path . exists ( self . path ) :
with open ( self . path , " r " , encoding = ' utf-8 ' ) as f : def add_recent_peer ( self , peer : LNPeerAddr ) :
raw = f . read ( ) addr = DBSession . query ( AddressInDB ) . filter_by ( node_id = peer . pubkey . hex ( ) ) . one_or_none ( )
self . data = json . loads ( raw ) if addr is None :
# channels addr = AddressInDB ( node_id = peer . pubkey . hex ( ) , host = peer . host , port = peer . port , last_connected_date = datetime . datetime . now ( ) )
channel_infos = self . get ( ' channel_infos ' , { } ) else :
for short_channel_id , channel_info_d in channel_infos . items ( ) : addr . last_connected_date = datetime . datetime . now ( )
channel_info = ChannelInfo . from_json ( channel_info_d ) DBSession . add ( addr )
short_channel_id = bfh ( short_channel_id ) DBSession . commit ( )
self . add_verified_channel_info ( short_channel_id , channel_info )
# nodes def get_200_randomly_sorted_nodes_not_in ( self , node_ids_bytes ) :
node_infos = self . get ( ' node_infos ' , { } ) unshuffled = DBSession \
for node_id , node_info_d in node_infos . items ( ) : . query ( NodeInfoInDB ) \
node_info = NodeInfo . from_json ( node_info_d ) . filter ( not_ ( NodeInfoInDB . node_id . in_ ( x . hex ( ) for x in node_ids_bytes ) ) ) \
node_id = bfh ( node_id ) . limit ( 200 ) \
self . nodes [ node_id ] = node_info . all ( )
# recent peers return random . sample ( unshuffled , len ( unshuffled ) )
recent_peers = self . get ( ' recent_peers ' , { } )
for host , port , pubkey in recent_peers : def nodes_get ( self , node_id ) :
peer = LNPeerAddr ( str ( host ) , int ( port ) , bfh ( pubkey ) ) return self . network . run_from_another_thread ( self . _nodes_get ( node_id ) )
self . _recent_peers . append ( peer )
# last good address async def _nodes_get ( self , node_id ) :
last_good_addr = self . get ( ' last_good_address ' , { } ) return DBSession \
for node_id , host_and_port in last_good_addr . items ( ) : . query ( NodeInfoInDB ) \
host , port = host_and_port . filter_by ( node_id = node_id . hex ( ) ) \
self . _last_good_address [ bfh ( node_id ) ] = LNPeerAddr ( str ( host ) , int ( port ) , bfh ( node_id ) ) . one_or_none ( )
def save_data ( self ) : def get_last_good_address ( self , node_id ) - > Optional [ LNPeerAddr ] :
with self . lock : adr_db = DBSession \
# channels . query ( AddressInDB ) \
channel_infos = { } . filter_by ( node_id = node_id . hex ( ) ) \
for short_channel_id , channel_info in self . _id_to_channel_info . items ( ) : . order_by ( AddressInDB . last_connected_date . desc ( ) ) \
channel_infos [ bh2u ( short_channel_id ) ] = channel_info . one_or_none ( )
self . put ( ' channel_infos ' , channel_infos ) if not adr_db :
# nodes return None
node_infos = { } return LNPeerAddr ( adr_db . host , adr_db . port , bytes . fromhex ( adr_db . node_id ) )
for node_id , node_info in self . nodes . items ( ) :
node_infos [ bh2u ( node_id ) ] = node_info def get_recent_peers ( self ) :
self . put ( ' node_infos ' , node_infos ) return [ LNPeerAddr ( x . host , x . port , bytes . fromhex ( x . node_id ) ) for x in DBSession \
# recent peers . query ( AddressInDB ) \
recent_peers = [ ] . select_from ( NodeInfoInDB ) \
for peer in self . _recent_peers : . order_by ( AddressInDB . last_connected_date . desc ( ) ) \
recent_peers . append ( . limit ( self . NUM_MAX_RECENT_PEERS ) ]
[ str ( peer . host ) , int ( peer . port ) , bh2u ( peer . pubkey ) ] )
self . put ( ' recent_peers ' , recent_peers ) def get_channel_info ( self , channel_id : bytes ) :
# last good address return self . chan_query_for_id ( channel_id ) . one_or_none ( )
last_good_addr = { }
for node_id , peer in self . _last_good_address . items ( ) :
last_good_addr [ bh2u ( node_id ) ] = [ str ( peer . host ) , int ( peer . port ) ]
self . put ( ' last_good_address ' , last_good_addr )
self . write ( )
def __len__ ( self ) :
# number of channels
return len ( self . _id_to_channel_info )
def capacity ( self ) :
# capacity of the network
return sum ( c . capacity_sat for c in self . _id_to_channel_info . values ( ) if c . capacity_sat is not None )
def get_channel_info ( self , channel_id : bytes ) - > Optional [ ChannelInfo ] :
return self . _id_to_channel_info . get ( channel_id , None )
def get_channels_for_node ( self , node_id ) : def get_channels_for_node ( self , node_id ) :
""" Returns the set of channels that have node_id as one of the endpoints. """ """ Returns the set of channels that have node_id as one of the endpoints. """
return self . _channels_for_node [ node_id ] condition = or_ (
ChannelInfoInDB . node1_id == node_id . hex ( ) ,
ChannelInfoInDB . node2_id == node_id . hex ( ) )
rows = DBSession . query ( ChannelInfoInDB ) . filter ( condition ) . all ( )
return [ bytes . fromhex ( x . short_channel_id ) for x in rows ]
def add_verified_channel_info ( self , short_id , capacity ) :
# called from lnchannelverifier
channel_info = self . get_channel_info ( short_id )
channel_info . trusted = True
channel_info . capacity = capacity
DBSession . commit ( )
def add_verified_channel_info ( self , short_channel_id : bytes , channel_info : ChannelInfo ) : @profiler
with self . lock : def on_channel_announcement ( self , msg_payloads , trusted = False ) :
self . _id_to_channel_info [ short_channel_id ] = channel_info if type ( msg_payloads ) is dict :
self . _channels_for_node [ channel_info . node_id_1 ] . add ( short_channel_id ) msg_payloads = [ msg_payloads ]
self . _channels_for_node [ channel_info . node_id_2 ] . add ( short_channel_id ) for msg in msg_payloads :
short_channel_id = msg [ ' short_channel_id ' ]
if DBSession . query ( ChannelInfoInDB ) . filter_by ( short_channel_id = bh2u ( short_channel_id ) ) . count ( ) :
continue
if constants . net . rev_genesis_bytes ( ) != msg [ ' chain_hash ' ] :
#self.print_error("ChanAnn has unexpected chain_hash {}".format(bh2u(msg_payload['chain_hash'])))
continue
try :
channel_info = ChannelInfoInDB . from_msg ( msg )
except UnknownEvenFeatureBits :
continue
channel_info . trusted = trusted
DBSession . add ( channel_info )
if not trusted : self . ca_verifier . add_new_channel_info ( channel_info . short_channel_id , channel_info . msg_payload )
DBSession . commit ( )
self . network . trigger_callback ( ' ln_status ' ) self . network . trigger_callback ( ' ln_status ' )
self . update_counts ( )
def get_recent_peers ( self ) : @profiler
with self . lock : def on_channel_update ( self , msg_payloads , trusted = False ) :
return list ( self . _recent_peers ) if type ( msg_payloads ) is dict :
msg_payloads = [ msg_payloads ]
def add_recent_peer ( self , peer : LNPeerAddr ) : short_channel_ids = [ msg_payload [ ' short_channel_id ' ] . hex ( ) for msg_payload in msg_payloads ]
with self . lock : channel_infos_list = DBSession . query ( ChannelInfoInDB ) . filter ( ChannelInfoInDB . short_channel_id . in_ ( short_channel_ids ) ) . all ( )
# list is ordered channel_infos = { bfh ( x . short_channel_id ) : x for x in channel_infos_list }
if peer in self . _recent_peers : for msg_payload in msg_payloads :
self . _recent_peers . remove ( peer ) short_channel_id = msg_payload [ ' short_channel_id ' ]
self . _recent_peers . insert ( 0 , peer ) if constants . net . rev_genesis_bytes ( ) != msg_payload [ ' chain_hash ' ] :
self . _recent_peers = self . _recent_peers [ : self . NUM_MAX_RECENT_PEERS ] continue
self . _last_good_address [ peer . pubkey ] = peer channel_info = channel_infos . get ( short_channel_id )
channel_info . on_channel_update ( msg_payload , trusted = trusted )
def get_last_good_address ( self , node_id : bytes ) - > Optional [ LNPeerAddr ] : DBSession . commit ( )
return self . _last_good_address . get ( node_id , None )
def on_channel_announcement ( self , msg_payload , trusted = False ) :
short_channel_id = msg_payload [ ' short_channel_id ' ]
if short_channel_id in self . _id_to_channel_info :
return
if constants . net . rev_genesis_bytes ( ) != msg_payload [ ' chain_hash ' ] :
#self.print_error("ChanAnn has unexpected chain_hash {}".format(bh2u(msg_payload['chain_hash'])))
return
try :
channel_info = ChannelInfo ( msg_payload )
except UnknownEvenFeatureBits :
return
if trusted :
self . add_verified_channel_info ( short_channel_id , channel_info )
else :
self . ca_verifier . add_new_channel_info ( channel_info )
def on_channel_update ( self , msg_payload , trusted = False ) : @profiler
short_channel_id = msg_payload [ ' short_channel_id ' ] def on_node_announcement ( self , msg_payloads ) :
if constants . net . rev_genesis_bytes ( ) != msg_payload [ ' chain_hash ' ] : if type ( msg_payloads ) is dict :
return msg_payloads = [ msg_payloads ]
# try finding channel in pending db addresses = DBSession . query ( AddressInDB ) . all ( )
channel_info = self . ca_verifier . get_pending_channel_info ( short_channel_id ) have_addr = { }
if channel_info is None : for addr in addresses :
# try finding channel in verified db have_addr [ ( addr . node_id , addr . host , addr . port ) ] = addr
channel_info = self . _id_to_channel_info . get ( short_channel_id , None )
if channel_info is None : nodes = DBSession . query ( NodeInfoInDB ) . all ( )
self . print_error ( " could not find " , short_channel_id ) timestamps = { }
raise NotFoundChanAnnouncementForUpdate ( ) for node in nodes :
channel_info . on_channel_update ( msg_payload , trusted = trusted ) no_millisecs = node . timestamp [ : len ( " 0000-00-00 00:00:00 " ) ]
timestamps [ bfh ( node . node_id ) ] = datetime . datetime . strptime ( no_millisecs , " % Y- % m- %d % H: % M: % S " )
def on_node_announcement ( self , msg_payload ) : old_addr = None
pubkey = msg_payload [ ' node_id ' ] for msg_payload in msg_payloads :
signature = msg_payload [ ' signature ' ] pubkey = msg_payload [ ' node_id ' ]
h = sha256d ( msg_payload [ ' raw ' ] [ 66 : ] ) signature = msg_payload [ ' signature ' ]
if not ecc . verify_signature ( pubkey , signature , h ) : h = sha256d ( msg_payload [ ' raw ' ] [ 66 : ] )
return if not ecc . verify_signature ( pubkey , signature , h ) :
old_node_info = self . nodes . get ( pubkey , None ) continue
try : try :
new_node_info = NodeInfo ( msg_payload ) new_node_info , addresses = NodeInfoInDB . from_msg ( msg_payload )
except UnknownEvenFeatureBits : except UnknownEvenFeatureBits :
return continue
# TODO if this message is for a new node, and if we have no associated if timestamps . get ( pubkey ) and timestamps [ pubkey ] > = new_node_info . timestamp :
# channels for this node, we should ignore the message and return here, continue # ignore
# to mitigate DOS. but race condition: the channels we have for this DBSession . add ( new_node_info )
# node, might be under verification in self.ca_verifier, what then? for new_addr in addresses :
if old_node_info and old_node_info . timestamp > = new_node_info . timestamp : key = ( new_addr . node_id , new_addr . host , new_addr . port )
return # ignore old_addr = have_addr . get ( key )
self . nodes [ pubkey ] = new_node_info if old_addr :
# since old_addr is embedded in have_addr,
# it will still live when commmit is called
old_addr . last_connected_date = new_addr . last_connected_date
del new_addr
else :
DBSession . add ( new_addr )
have_addr [ key ] = new_addr
# TODO if this message is for a new node, and if we have no associated
# channels for this node, we should ignore the message and return here,
# to mitigate DOS. but race condition: the channels we have for this
# node, might be under verification in self.ca_verifier, what then?
del nodes , addresses
if old_addr :
del old_addr
DBSession . commit ( )
self . network . trigger_callback ( ' ln_status ' )
self . update_counts ( )
def get_routing_policy_for_channel ( self , start_node_id : bytes , def get_routing_policy_for_channel ( self , start_node_id : bytes ,
short_channel_id : bytes ) - > Optional [ ChannelInfoDirectedPolicy ] : short_channel_id : bytes ) - > Optional [ bytes ] :
if not start_node_id or not short_channel_id : return None if not start_node_id or not short_channel_id : return None
channel_info = self . get_channel_info ( short_channel_id ) channel_info = self . get_channel_info ( short_channel_id )
if channel_info is not None : if channel_info is not None :
return channel_info . get_policy_for_node ( start_node_id ) return channel_info . get_policy_for_node ( start_node_id )
return self . _channel_updates_for_private_channels . get ( ( start_node_id , short_channel_id ) ) msg = self . _channel_updates_for_private_channels . get ( ( start_node_id , short_channel_id ) )
if not msg : return None
return Policy . from_msg ( msg , None , short_channel_id ) # won't actually be written to DB
def add_channel_update_for_private_channel ( self , msg_payload : dict , start_node_id : bytes ) : def add_channel_update_for_private_channel ( self , msg_payload : dict , start_node_id : bytes ) :
if not verify_sig_for_channel_update ( msg_payload , start_node_id ) : if not verify_sig_for_channel_update ( msg_payload , start_node_id ) :
return # ignore return # ignore
short_channel_id = msg_payload [ ' short_channel_id ' ] short_channel_id = msg_payload [ ' short_channel_id ' ]
policy = ChannelInfoDirectedPolicy ( msg_payload ) self . _channel_updates_for_private_channels [ ( start_node_id , short_channel_id ) ] = msg_payload
self . _channel_updates_for_private_channels [ ( start_node_id , short_channel_id ) ] = policy
def remove_channel ( self , short_channel_id ) : def remove_channel ( self , short_channel_id ) :
try : self . chan_query_for_id ( short_channel_id ) . delete ( ' evaluate ' )
channel_info = self . _id_to_channel_info [ short_channel_id ] DBSession . commit ( )
except KeyError :
self . print_error ( f ' remove_channel: cannot find channel { bh2u ( short_channel_id ) } ' ) def chan_query_for_id ( self , short_channel_id ) - > Query :
return return DBSession . query ( ChannelInfoInDB ) . filter_by ( short_channel_id = short_channel_id . hex ( ) )
self . _id_to_channel_info . pop ( short_channel_id , None )
for node in ( channel_info . node_id_1 , channel_info . node_id_2 ) :
try :
self . _channels_for_node [ node ] . remove ( short_channel_id )
except KeyError :
pass
def print_graph ( self , full_ids = False ) : def print_graph ( self , full_ids = False ) :
# used for debugging. # used for debugging.
# FIXME there is a race here - iterables could change size from another thread # FIXME there is a race here - iterables could change size from another thread
def other_node_id ( node_id , channel_id ) : def other_node_id ( node_id , channel_id ) :
channel_info = self . _id_to_channel_info [ channel_id ] channel_info = self . get_channel_info ( channel_id )
if node_id == channel_info . node_id_1 : if node_id == channel_info . node1 _id :
other = channel_info . node_id_2 other = channel_info . node2 _id
else : else :
other = channel_info . node_id_1 other = channel_info . node1 _id
return other if full_ids else other [ - 4 : ] return other if full_ids else other [ - 4 : ]
self . print_msg ( ' node: { (channel, other_node), ...} ' ) self . print_msg ( ' nodes ' )
for node_id , short_channel_ids in list ( self . _channels_for_node . items ( ) ) : for node in DBSession . query ( NodeInfoInDB ) . all ( ) :
short_channel_ids = { ( bh2u ( cid ) , bh2u ( other_node_id ( node_id , cid ) ) ) self . print_msg ( node )
for cid in short_channel_ids }
node_id = bh2u ( node_id ) if full_ids else bh2u ( node_id [ - 4 : ] ) self . print_msg ( ' channels ' )
self . print_msg ( ' {} : {} ' . format ( node_id , short_channel_ids ) ) for channel_info in DBSession . query ( ChannelInfoInDB ) . all ( ) :
node1 = channel_info . node1_id
self . print_msg ( ' channel: node1, node2, direction ' ) node2 = channel_info . node2_id
for short_channel_id , channel_info in list ( self . _id_to_channel_info . items ( ) ) :
node1 = channel_info . node_id_1
node2 = channel_info . node_id_2
direction1 = channel_info . get_policy_for_node ( node1 ) is not None direction1 = channel_info . get_policy_for_node ( node1 ) is not None
direction2 = channel_info . get_policy_for_node ( node2 ) is not None direction2 = channel_info . get_policy_for_node ( node2 ) is not None
if direction1 and direction2 : if direction1 and direction2 :
@ -514,8 +506,10 @@ class RouteEdge(NamedTuple("RouteEdge", [('node_id', bytes),
+ ( amount_msat * self . fee_proportional_millionths / / 1_000_000 ) + ( amount_msat * self . fee_proportional_millionths / / 1_000_000 )
@classmethod @classmethod
def from_channel_policy ( cls , channel_policy : ChannelInfoDirectedPolicy , def from_channel_policy ( cls , channel_policy : ' Policy ' ,
short_channel_id : bytes , end_node : bytes ) - > ' RouteEdge ' : short_channel_id : bytes , end_node : bytes ) - > ' RouteEdge ' :
assert type ( short_channel_id ) is bytes
assert type ( end_node ) is bytes
return RouteEdge ( end_node , return RouteEdge ( end_node ,
short_channel_id , short_channel_id ,
channel_policy . fee_base_msat , channel_policy . fee_base_msat ,
@ -582,7 +576,7 @@ class LNPathFinder(PrintError):
channel_policy = channel_info . get_policy_for_node ( start_node ) channel_policy = channel_info . get_policy_for_node ( start_node )
if channel_policy is None : return float ( ' inf ' ) , 0 if channel_policy is None : return float ( ' inf ' ) , 0
if channel_policy . disabled : return float ( ' inf ' ) , 0 if channel_policy . is_ disabled( ) : return float ( ' inf ' ) , 0
route_edge = RouteEdge . from_channel_policy ( channel_policy , short_channel_id , end_node ) route_edge = RouteEdge . from_channel_policy ( channel_policy , short_channel_id , end_node )
if payment_amt_msat < channel_policy . htlc_minimum_msat : if payment_amt_msat < channel_policy . htlc_minimum_msat :
return float ( ' inf ' ) , 0 # payment amount too little return float ( ' inf ' ) , 0 # payment amount too little
@ -611,6 +605,8 @@ class LNPathFinder(PrintError):
To get from node ret [ n ] [ 0 ] to ret [ n + 1 ] [ 0 ] , use channel ret [ n + 1 ] [ 1 ] ; To get from node ret [ n ] [ 0 ] to ret [ n + 1 ] [ 0 ] , use channel ret [ n + 1 ] [ 1 ] ;
i . e . an element reads as , " to get to node_id, travel through short_channel_id " i . e . an element reads as , " to get to node_id, travel through short_channel_id "
""" """
assert type ( nodeA ) is bytes
assert type ( nodeB ) is bytes
assert type ( invoice_amount_msat ) is int assert type ( invoice_amount_msat ) is int
if my_channels is None : my_channels = [ ] if my_channels is None : my_channels = [ ]
my_channels = { chan . short_channel_id : chan for chan in my_channels } my_channels = { chan . short_channel_id : chan for chan in my_channels }
@ -657,9 +653,10 @@ class LNPathFinder(PrintError):
# so there are duplicates in the queue, that we discard now: # so there are duplicates in the queue, that we discard now:
continue continue
for edge_channel_id in self . channel_db . get_channels_for_node ( edge_endnode ) : for edge_channel_id in self . channel_db . get_channels_for_node ( edge_endnode ) :
assert type ( edge_channel_id ) is bytes
if edge_channel_id in self . blacklist : continue if edge_channel_id in self . blacklist : continue
channel_info = self . channel_db . get_channel_info ( edge_channel_id ) channel_info = self . channel_db . get_channel_info ( edge_channel_id )
edge_startnode = channel_info . node_id_2 if channel_info . node_id_1 == edge_endnode else channel_info . node_id_1 edge_startnode = bfh ( channel_info . node2_id ) if bfh ( channel_info . node1_id ) == edge_endnode else bfh ( channel_info . node1_id )
inspect_edge ( ) inspect_edge ( )
else : else :
return None # no path found return None # no path found
@ -682,7 +679,7 @@ class LNPathFinder(PrintError):
for node_id , short_channel_id in path : for node_id , short_channel_id in path :
channel_policy = self . channel_db . get_routing_policy_for_channel ( prev_node_id , short_channel_id ) channel_policy = self . channel_db . get_routing_policy_for_channel ( prev_node_id , short_channel_id )
if channel_policy is None : if channel_policy is None :
raise Exception ( f ' cannot find channel policy for short_channel_id: { bh2u ( short_channel_id ) } ' ) raise NoChannelPolicy ( short_channel_id )
route . append ( RouteEdge . from_channel_policy ( channel_policy , short_channel_id , node_id ) ) route . append ( RouteEdge . from_channel_policy ( channel_policy , short_channel_id , node_id ) )
prev_node_id = node_id prev_node_id = node_id
return route return route
Janus
7 years ago
committed by
ThomasV