From 6e46a63c5738798eb303698d34833cbbe7038196 Mon Sep 17 00:00:00 2001 From: Michael Schmoock Date: Sat, 25 Feb 2023 23:18:56 +0100 Subject: [PATCH] pygossmap: adds statistic and filter module Includes a lot of useful filters and statistical methods. To see a gossip_store summary: ``` s = GossmapStats(g) s.print_stats() ``` --- contrib/pyln-client/pyln/client/__init__.py | 2 + contrib/pyln-client/pyln/client/gossmap.py | 46 ++++ .../pyln-client/pyln/client/gossmapstats.py | 208 ++++++++++++++++++ 3 files changed, 256 insertions(+) create mode 100644 contrib/pyln-client/pyln/client/gossmapstats.py diff --git a/contrib/pyln-client/pyln/client/__init__.py b/contrib/pyln-client/pyln/client/__init__.py index 07064f909..710c1e56a 100644 --- a/contrib/pyln-client/pyln/client/__init__.py +++ b/contrib/pyln-client/pyln/client/__init__.py @@ -1,6 +1,7 @@ from .lightning import LightningRpc, RpcError, Millisatoshi from .plugin import Plugin, monkey_patch, RpcException from .gossmap import Gossmap, GossmapNode, GossmapChannel, GossmapHalfchannel, GossmapNodeId, LnFeatureBits +from .gossmapstats import GossmapStats __version__ = "23.02" @@ -18,4 +19,5 @@ __all__ = [ "GossmapHalfchannel", "GossmapNodeId", "LnFeatureBits", + "GossmapStats", ] diff --git a/contrib/pyln-client/pyln/client/gossmap.py b/contrib/pyln-client/pyln/client/gossmap.py index 21395fdef..c290d1183 100755 --- a/contrib/pyln-client/pyln/client/gossmap.py +++ b/contrib/pyln-client/pyln/client/gossmap.py @@ -9,6 +9,7 @@ import io import base64 import socket import struct +import time # These duplicate constants in lightning/common/gossip_store.h GOSSIP_STORE_MAJOR_VERSION = (0 << 5) @@ -238,6 +239,10 @@ class GossmapChannel(object): return False return True + def is_tor_only(c): + """ Checks if a channel has TOR only nodes on both ends """ + return c.node1.is_tor_only() and c.node2.is_tor_only() + class GossmapNode(object): """A node: fields of node_announcement are in .fields, @@ -344,6 +349,45 @@ class GossmapNode(object): return 'ipv6' return 'dns' + def has_clearnet(self): + """ Checks if a node has one or more clearnet addresses """ + if not self.announced or len(self.addresses) == 0: + return False + for i in range(len(self.addresses)): + if self.get_address_type(i) != 'tor': + return True + return False + + def has_tor(self): + """ Checks if a node has one or more TOR addresses """ + if not self.announced or len(self.addresses) == 0: + return False + for i in range(len(self.addresses)): + if self.get_address_type(i) == 'tor': + return True + return False + + def is_tor_only(self): + """ Checks if a node has only TOR and no addresses announced """ + if not self.announced or len(self.addresses) == 0: + return False + for i in range(len(self.addresses)): + if self.get_address_type(i) != 'tor': + return False + return True + + def is_tor_strict(self): + """ Checks if a node is TOR only + and is not publicly connected to any non-TOR nodes """ + if not self.is_tor_only(): + return False + for c in self.channels: + other = c.node1 if self != c.node1 else c.node2 + if other.has_tor(): + continue + return False + return True + class Gossmap(object): """Class to represent the gossip map of the network""" @@ -572,6 +616,7 @@ class Gossmap(object): def refresh(self): """Catch up with any changes to the gossip store""" + start_time = time.time() while True: rec, hdr = self._read_record() if rec is None: # EOF @@ -602,3 +647,4 @@ class Gossmap(object): self.reopen_store() else: continue + self.processing_time += time.time() - start_time diff --git a/contrib/pyln-client/pyln/client/gossmapstats.py b/contrib/pyln-client/pyln/client/gossmapstats.py new file mode 100644 index 000000000..5479e2dc9 --- /dev/null +++ b/contrib/pyln-client/pyln/client/gossmapstats.py @@ -0,0 +1,208 @@ +from pyln.client import Gossmap, GossmapChannel, GossmapNode, GossmapHalfchannel, LnFeatureBits +from typing import Iterable, List, Optional, Callable + +import operator +import statistics + + +class GossmapStats(object): + def __init__(self, g: Gossmap): + self.g = g + + # First the generic filter functions + def filter_nodes(self, predicate: Callable[[GossmapNode], bool], nodes: Optional[Iterable[GossmapNode]] = None) -> List[GossmapNode]: + """ Filter nodes using an arbitrary function or lamda predicate. """ + if nodes is None: + nodes = self.g.nodes.values() + return [n for n in nodes if predicate(n)] + + def filter_channels(self, predicate: Callable[[GossmapChannel], bool], channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapChannel]: + """ Filters channels using an arbitrary function or lambda predicate. """ + if channels is None: + channels = self.g.channels.values() + return [c for c in channels if predicate(c)] + + def filter_halfchannels(self, predicate: Callable[[GossmapHalfchannel], bool], channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapHalfchannel]: + """ Filters half-channels using an arbitrary function or lambda predicate. """ + if channels is None: + channels = self.g.channels.values() + hc0 = [c.half_channels[0] for c in channels if c.half_channels[0] is not None and predicate(c.half_channels[0])] + hc1 = [c.half_channels[1] for c in channels if c.half_channels[1] is not None and predicate(c.half_channels[1])] + return hc0 + hc1 + + # Now a bunch of predefined specific filter methods + def filter_nodes_ratelimited(self, nodes: Optional[Iterable[GossmapNode]] = None) -> List[GossmapNode]: + """ Filters nodes being marked by cln as ratelimited, when they send out too many updates. """ + return self.filter_nodes(lambda n: n.hdr is not None and n.hdr.ratelimit, nodes) + + def filter_nodes_unannounced(self, nodes: Optional[Iterable[GossmapNode]] = None) -> List[GossmapNode]: + """ Filters nodes that are only known by a channel, i.e. missing a node_announcement. + Usually happens when a peer has been offline for a while. """ + return self.filter_nodes(lambda n: not n.announced, nodes) + + def filter_nodes_feature(self, bit, nodes: Optional[Iterable[GossmapNode]] = None) -> List[GossmapNode]: + """Filters nodes based on node_announcement feature bits. """ + return self.filter_nodes(lambda n: n.announced and 3 << bit & n.features != 0, nodes) + + def filter_nodes_feature_compulsory(self, bit, nodes: Optional[Iterable[GossmapNode]] = None) -> List[GossmapNode]: + """Filters nodes based on node_announcement feature bits. """ + return self.filter_nodes(lambda n: n.announced and 1 << bit & n.features != 0, nodes) + + def filter_nodes_feature_optional(self, bit, nodes: Optional[Iterable[GossmapNode]] = None) -> List[GossmapNode]: + """Filters nodes based on node_announcement feature bits. """ + return self.filter_nodes(lambda n: n.announced and 2 << bit & n.features != 0, nodes) + + def filter_nodes_address_type(self, typestr, nodes: Optional[Iterable[GossmapNode]] = None) -> List[GossmapNode]: + """ Filters nodes having at least one address of typetr: 'ipv4', 'ipv6', 'tor' or 'dns'. """ + return self.filter_nodes(lambda n: n.announced and len([idx for idx in range(len(n.addresses)) if n.get_address_type(idx) == typestr]) > 0, nodes) + + def filter_nodes_tor_only(self, nodes: Optional[Iterable[GossmapNode]] = None) -> List[GossmapNode]: + """ Filters nodes that only announce TOR addresses, if any. """ + return self.filter_nodes(lambda n: n.is_tor_only(), nodes) + + def filter_nodes_tor_strict(self, nodes: Optional[Iterable[GossmapNode]] = None) -> List[GossmapNode]: + """ Filters TOR only nodes that don't (or possibly can't) connect to non-TOR nodes. """ + return self.filter_nodes(lambda n: n.is_tor_strict(), nodes) + + def filter_nodes_no_addresses(self, nodes: Optional[Iterable[GossmapNode]] = None) -> List[GossmapNode]: + """ Filters nodes that don't announce any addresses. """ + return self.filter_nodes(lambda n: n.announced and len(n.addresses) == 0, nodes) + + def filter_nodes_channel_count(self, count, op=operator.ge, nodes: Optional[Iterable[GossmapNode]] = None) -> List[GossmapNode]: + """ Filters nodes by its channel count (default op: being greater or eaqual). """ + return self.filter_nodes(lambda n: op(len(n.channels), count), nodes) + + def filter_channels_feature(self, bit, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapChannel]: + """ Filters channels based on channel_announcement feature bits. """ + return self.filter_channels(lambda c: 3 << bit & c.features != 0, channels) + + def filter_channels_feature_compulsory(self, bit, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapChannel]: + """ Filters channels based on channel_announcement feature bits. """ + return self.filter_channels(lambda c: 1 << bit & c.features != 0, channels) + + def filter_channels_feature_optional(self, bit, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapChannel]: + """ Filters channels based on channel_announcement feature bits. """ + return self.filter_channels(lambda c: 2 << bit & c.features != 0, channels) + + def filter_channels_unidirectional(self, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapChannel]: + """ Filters channels that are known only in one direction, i.e. other peer seems offline for a long time. """ + return self.filter_channels(lambda c: c.half_channels[0] is None or c.half_channels[1] is None, channels) + + def filter_channels_nosatoshis(self, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapChannel]: + """ Filters channels with missing WIRE_GOSSIP_STORE_CHANNEL_AMOUNT. This should not happen. """ + return self.filter_channels(lambda c: c.satoshis is None, channels) + + def filter_channels_tor_only(self, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapChannel]: + """ Filters all channels that are connected to TOR only nodes on both ends. """ + return self.filter_channels(lambda c: c.is_tor_only(), channels) + + def filter_channels_capacity(self, satoshis, op=operator.ge, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapChannel]: + """ Filter channels by its capacity (default op: being greater or equal). """ + return self.filter_channels(lambda c: c.satoshis is not None and op(c.satoshis, satoshis), channels) + + def filter_channels_disabled_bidirectional(self, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapChannel]: + """ Filters channels that are disabled in both directions. """ + return self.filter_channels(lambda c: c.half_channels[0] is not None and c.half_channels[0].disabled and c.half_channels[1] is not None and c.half_channels[1].disabled, channels) + + def filter_channels_disabled_unidirectional(self, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapChannel]: + """ Filters channels that are disabled only in one direction. """ + if channels is None: + channels = self.g.channels.values() + hc0 = [c for c in channels if c.half_channels[0] is not None and c.half_channels[0].disabled and (c.half_channels[1] is None or not c.half_channels[1].disabled)] + hc1 = [c for c in channels if c.half_channels[1] is not None and c.half_channels[1].disabled and (c.half_channels[0] is None or not c.half_channels[0].disabled)] + return hc0 + hc1 + + def filter_halfchannels_fee_base(self, msat, op=operator.le, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapHalfchannel]: + """ Filters half-channels by its base fee (default op: being lower or equal). """ + return self.filter_halfchannels(lambda hc: op(hc.fee_base_msat, msat), channels) + + def filter_halfchannels_fee_ppm(self, msat, op=operator.le, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapHalfchannel]: + """ Filters half-channels by its ppm fee (default op: being lower or equal). """ + return self.filter_halfchannels(lambda hc: op(hc.fee_proportional_millionths, msat), channels) + + def filter_halfchannels_disabled(self, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapHalfchannel]: + """ Filters half-channels that are disabled. """ + return self.filter_halfchannels(lambda hc: hc.disabled, channels) + + def filter_halfchannels_ratelimited(self, channels: Optional[Iterable[GossmapChannel]] = None) -> List[GossmapHalfchannel]: + """ Filters half-channels that are being marked as ratelimited for sending out too many updates. """ + return self.filter_halfchannels(lambda hc: hc.hdr.ratelimit, channels) + + def quantiles_nodes_channel_count(self, tiles=100, nodes: Optional[Iterable[GossmapNode]] = None) -> List[float]: + if nodes is None: + nodes = self.g.nodes.values() + return statistics.quantiles([len(n.channels) for n in nodes], n=tiles) + + def quantiles_channels_capacity(self, tiles=100, channels: Optional[Iterable[GossmapChannel]] = None) -> List[float]: + if channels is None: + channels = self.g.channels.values() + return statistics.quantiles([c.satoshis for c in channels if c.satoshis is not None], n=tiles) + + def quantiles_halfchannels_fee_base(self, tiles=100, channels: Optional[Iterable[GossmapChannel]] = None) -> List[float]: + if channels is None: + channels = self.g.channels.values() + hc0 = [c.half_channels[0].fee_base_msat for c in channels if c.half_channels[0] is not None] + hc1 = [c.half_channels[1].fee_base_msat for c in channels if c.half_channels[1] is not None] + return statistics.quantiles(hc0 + hc1, n=tiles) + + def quantiles_halfchannels_fee_ppm(self, tiles=100, channels: Optional[Iterable[GossmapChannel]] = None) -> List[float]: + if channels is None: + channels = self.g.channels.values() + hc0 = [c.half_channels[0].fee_proportional_millionths for c in channels if c.half_channels[0] is not None] + hc1 = [c.half_channels[1].fee_proportional_millionths for c in channels if c.half_channels[1] is not None] + return statistics.quantiles(hc0 + hc1, n=tiles) + + def print_stats(self): + print("#### pyln-client gossmap stats ####") + print(f"The gossip_store has a total of {len(self.g.nodes)} nodes and {len(self.g.channels)} channels.") + print(f"Total processing time was {self.g.processing_time} seconds.") + print("") + + print("CONSISTENCY") + print(f" - {len(self.filter_nodes_unannounced())} orphan nodes without a node_announcement, only known from a channel_announcement.") + print(f" - {len(self.g.orphan_channel_updates)} orphan channel_updates without a prior channel_announcement.") + print(f" - {len(self.filter_nodes_ratelimited())} nodes marked as ratelimited. (sending too many updates).") + print(f" - {len(self.filter_halfchannels_ratelimited())} half-channels marked as ratelimited. (sending too many updates).") + print(f" - {len(self.filter_channels_nosatoshis())} channels without capacity (missing WIRE_GOSSIP_STORE_CHANNEL_AMOUNT). Should be 0.") + print("") + + print("STRUCTURE") + print(f" - {len(self.filter_channels_unidirectional())} channels that are known only in one direction, other peer seems offline for a long time.") + print(f" - {len(self.filter_halfchannels_disabled())} total disabled half-channels.") + print(f" - {len(self.filter_channels_disabled_unidirectional())} channels are only disabled in one direction.") + print(f" - {len(self.filter_channels_disabled_bidirectional())} channels are disabled in both directions.") + print(f" - channel_count per node quantiles(10): {self.quantiles_nodes_channel_count(10)}.") + print(f" - channel_capacity quantiles(10): {self.quantiles_channels_capacity(10)}.") + print("") + + print("ADDRESSES") + print(f" - {len(self.filter_nodes_address_type('ipv4'))} nodes announce IPv4 addresses.") + print(f" - {len(self.filter_nodes_address_type('ipv6'))} nodes announce IPv6 addresses.") + print(f" - {len(self.filter_nodes_address_type('tor'))} nodes announce TOR addresses.") + print(f" - {len(self.filter_nodes_address_type('dns'))} nodes announce DNS addresses.") + print(f" - {len(self.filter_nodes_no_addresses())} don't announce any address.") + print(f" - {len(self.filter_nodes_tor_only())} nodes announce only TOR addresses, if any.") + print(f" - {len(self.filter_nodes_tor_strict())} nodes announce only TOR addresses and don't, or possibly can't, connect to non-TOR nodes.") + print(f" - {len(self.filter_channels_tor_only())} channels are connected TOR only nodes on both ends.") + print("") + + print("FEES") + print(f" - {len(self.filter_halfchannels_fee_base(0))} half-channels have a base_fee of 0msat.") + print(f" - {len(self.filter_halfchannels_fee_base(1000, operator.ge))} half-channels have a base_fee >= 1000msat.") + print(f" - {len(self.filter_halfchannels_fee_ppm(0))} half-channels have a ppm_fee of 0.") + print(f" - {len(self.filter_halfchannels_fee_ppm(1000, operator.ge))} half-channels have a ppm_fee >= 1000.") + print(f" - base_fee quantiles(10): {self.quantiles_halfchannels_fee_base(10)}.") + print(f" - ppm_fee quantiles(10): {self.quantiles_halfchannels_fee_ppm(10)}.") + print("") + + print("FEATURES") + print(f" - {len(self.filter_nodes_feature_compulsory(LnFeatureBits.OPTION_DATA_LOSS_PROTECT))} nodes require data loss protection.") + print(f" - {len(self.filter_nodes_feature(LnFeatureBits.GOSSIP_QUERIES))} nodes support gossip queries.") + print(f" - {len(self.filter_nodes_feature(LnFeatureBits.GOSSIP_QUERIES_EX))} nodes support extended gossip queries.") + print(f" - {len(self.filter_nodes_feature(LnFeatureBits.BASIC_MPP))} nodes support basic MPP.") + print(f" - {len(self.filter_nodes_feature(LnFeatureBits.OPTION_ANCHOR_OUTPUTS))} nodes support anchor outputs.") + print(f" - {len(self.filter_nodes_feature(LnFeatureBits.OPTION_SCID_ALIAS))} nodes support scid alias.") + print(f" - {len(self.filter_nodes_feature(LnFeatureBits.OPTION_ZEROCONF))} nodes support zeroconf.") + print("") + + print("#### pyln-client gossmap END ####")