From cf936d296eac3f9649dfd94d8fb250a637816252 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 7 Aug 2024 10:15:55 +0930 Subject: [PATCH] devtools: add gossmap-compress to give minimal representation of gossmap topology. Simple format, which doesn't include node information, just the channels. Example: ``` $ ls -l gossip-store-2024-06-26 -rw------- 1 rusty rusty 98815543 Jul 26 09:47 gossip-store-2024-06-26 $ ./devtools/gossmap-compress -v compress gossip-store-2024-06-26 compressed 18693 nodes 61437 channels 46148 disabled channels (32620 no update) 9690 unique capacities 85 unique htlc_min 6867 unique htlc_max 807 unique basefee 2521 unique propfee 94 unique delay $ ls -l compressed -rw-rw-r-- 1 rusty rusty 1643258 Jul 26 09:51 compressed ``` Signed-off-by: Rusty Russell --- devtools/Makefile | 4 +- devtools/gossmap-compress.c | 363 ++++++++++++++++++++++++++++++++++++ 2 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 devtools/gossmap-compress.c diff --git a/devtools/Makefile b/devtools/Makefile index 3632b5476..cb2769c84 100644 --- a/devtools/Makefile +++ b/devtools/Makefile @@ -1,4 +1,4 @@ -DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore devtools/gossipwith devtools/create-gossipstore devtools/mkcommit devtools/mkfunding devtools/mkclose devtools/mkgossip devtools/mkencoded devtools/mkquery devtools/lightning-checkmessage devtools/topology devtools/route devtools/bolt12-cli devtools/encodeaddr devtools/features devtools/fp16 devtools/rune +DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore devtools/gossipwith devtools/create-gossipstore devtools/mkcommit devtools/mkfunding devtools/mkclose devtools/mkgossip devtools/mkencoded devtools/mkquery devtools/lightning-checkmessage devtools/topology devtools/route devtools/bolt12-cli devtools/encodeaddr devtools/features devtools/fp16 devtools/rune devtools/gossmap-compress ifeq ($(HAVE_SQLITE3),1) DEVTOOLS += devtools/checkchannels endif @@ -57,6 +57,8 @@ devtools/bolt11-cli: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(BITCOIN_OBJS) wire/f devtools/encodeaddr: common/utils.o common/bech32.o devtools/encodeaddr.o +devtools/gossmap-compress: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/tlvstream.o common/gossmap.o common/fp16.o devtools/gossmap-compress.o + devtools/bolt12-cli: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) wire/bolt12_wiregen.o wire/fromwire.o wire/towire.o common/bolt12.o common/bolt12_merkle.o devtools/bolt12-cli.o common/setup.o common/iso4217.o devtools/decodemsg: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(BITCOIN_OBJS) $(WIRE_PRINT_OBJS) wire/fromwire.o wire/towire.o devtools/print_wire.o devtools/decodemsg.o diff --git a/devtools/gossmap-compress.c b/devtools/gossmap-compress.c new file mode 100644 index 000000000..a81146df0 --- /dev/null +++ b/devtools/gossmap-compress.c @@ -0,0 +1,363 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool verbose = false; + +/* All {numbers} are bigsize. + * + * :=
+ *
:= "GOSSMAP_COMPRESSv1\0" + * := {channel_count} {start_nodeidx}*{channel_count} {end_nodeidx}*{channel_count} + * This describes each attached channel, eg if there are two + * channels, node 0 to node 1 and node 0 to node 2, this would be: + * 2 0 0 1 2 + * + * := * {channel_count*2} + * := {chanidx}*2+{direction} + * Selection of disabled channels and directions, expected to only be a few. Indexes into the + * first channel_ends array. Terminated by invalid index. + * + * := {channel_count}*{capacity_idx} + * := {capacity_count} {channel_count}*{capacity} + * This is one satoshi amount per channel. + * + * := {channel_count*2}*{htlc_min_idx} + * := {htlc_min_count} {htlc_min_count}*{htlc_min} + * These templates are all of the same form. A set of values, followed by + * an index into these values for each direction of each channel, in order + * 1. 0'th channel 1st direction + * 2. 0'th channel 2nd direction + * 3. 1'st channel 1st direction + * 4. 1'st channel 2nd direction + * + * := {channel_count*2}*{htlc_max_idx} + * Note that values 0 and 1 are special: 0 == channel capacity, 1 == 0.99 * channel capacity. + * := {htlc_max_count} {htlc_max_count}*{htlc_max} + * := {channel_count*2}*{basefee_idx} + * := {basefee_count} {basefee_count}*{basefee} + * := {channel_count*2}*{propfee_idx} + * := {propfee_count} {propfee_count}*{propfee} + * := {channel_count*2}*{delay_idx} + * := {delay_count} {delay_count}*{delay} + */ + +#define GC_HEADER "GOSSMAP_COMPRESSv1" +#define GC_HEADERLEN (sizeof(GC_HEADER)) + +static int cmp_node_num_chans(struct gossmap_node *const *a, + struct gossmap_node *const *b, + void *unused) +{ + return (int)(*a)->num_chans - (int)(*b)->num_chans; +} + +static void write_bigsize(int outfd, u64 val) +{ + u8 buf[BIGSIZE_MAX_LEN]; + size_t len; + + len = bigsize_put(buf, val); + if (!write_all(outfd, buf, len)) + errx(1, "Writing bigsize"); +} + +static int cmp_u64(const u64 *a, + const u64 *b, + void *unused) +{ + if (*a > *b) + return 1; + else if (*a < *b) + return -1; + return 0; +} + +static const u64 *deduplicate(const tal_t *ctx, const u64 *vals) +{ + u64 *sorted; + u64 *dedup; + size_t n; + + /* Sort and remove dups */ + sorted = tal_dup_talarr(tmpctx, u64, vals); + asort(sorted, tal_count(sorted), cmp_u64, NULL); + + dedup = tal_arr(ctx, u64, tal_count(sorted)); + n = 0; + dedup[n++] = sorted[0]; + for (size_t i = 1; i < tal_count(sorted); i++) { + if (sorted[i] == dedup[n-1]) + continue; + dedup[n++] = sorted[i]; + } + tal_resize(&dedup, n); + + return dedup; +} + +static size_t find_index(const u64 *template, u64 val) +{ + for (size_t i = 0; i < tal_count(template); i++) { + if (template[i] == val) + return i; + } + abort(); +} + +/* All templates are of the same form. Output all the distinct values, then + * write out which one is used by each channel */ +static void write_template_and_values(int outfd, const u64 *vals, const char *what) +{ + /* Sort and remove dups */ + const u64 *template = deduplicate(tmpctx, vals); + + if (verbose) + printf("%zu unique %s\n", tal_count(template), what); + + assert(tal_count(vals) >= tal_count(template)); + + /* Write template. */ + write_bigsize(outfd, tal_count(template)); + for (size_t i = 0; i < tal_count(template); i++) + write_bigsize(outfd, template[i]); + + /* Tie every channel into the template. O(N^2) but who + * cares? */ + for (size_t i = 0; i < tal_count(vals); i++) { + write_bigsize(outfd, find_index(template, vals[i])); + } +} + +static void write_bidir_perchan(int outfd, + struct gossmap *gossmap, + struct gossmap_chan **chans, + u64 (*get_value)(struct gossmap *, + const struct gossmap_chan *, + int), + const char *what) +{ + u64 *vals = tal_arr(tmpctx, u64, tal_count(chans) * 2); + + for (size_t i = 0; i < tal_count(chans); i++) { + for (size_t dir = 0; dir < 2; dir++) { + if (chans[i]->half[dir].enabled) + vals[i*2+dir] = get_value(gossmap, chans[i], dir); + else + vals[i*2+dir] = 0; + } + } + + write_template_and_values(outfd, vals, what); +} + +static u64 get_htlc_min(struct gossmap *gossmap, + const struct gossmap_chan *chan, + int dir) +{ + struct amount_msat msat; + gossmap_chan_get_update_details(gossmap, chan, dir, + NULL, NULL, NULL, NULL, NULL, &msat, NULL); + return msat.millisatoshis; /* Raw: compressed format */ +} + +static u64 get_htlc_max(struct gossmap *gossmap, + const struct gossmap_chan *chan, + int dir) +{ + struct amount_msat msat, capacity_msat; + struct amount_sat capacity_sats; + gossmap_chan_get_capacity(gossmap, chan, &capacity_sats); + gossmap_chan_get_update_details(gossmap, chan, dir, + NULL, NULL, NULL, NULL, NULL, NULL, &msat); + + /* Special value for the common case of "max_htlc == capacity" */ + if (amount_msat_eq_sat(msat, capacity_sats)) { + return 0; + } + /* Other common case: "max_htlc == 99% capacity" */ + if (amount_sat_to_msat(&capacity_msat, capacity_sats) + && amount_msat_scale(&capacity_msat, capacity_msat, 0.99) + && amount_msat_eq(msat, capacity_msat)) { + return 1; + } + return msat.millisatoshis; /* Raw: compressed format */ +} + +static u64 get_basefee(struct gossmap *gossmap, + const struct gossmap_chan *chan, + int dir) +{ + u32 basefee; + gossmap_chan_get_update_details(gossmap, chan, dir, + NULL, NULL, NULL, &basefee, NULL, NULL, NULL); + return basefee; +} + +static u64 get_propfee(struct gossmap *gossmap, + const struct gossmap_chan *chan, + int dir) +{ + u32 propfee; + gossmap_chan_get_update_details(gossmap, chan, dir, + NULL, NULL, NULL, NULL, &propfee, NULL, NULL); + return propfee; +} + +static u64 get_delay(struct gossmap *gossmap, + const struct gossmap_chan *chan, + int dir) +{ + return chan->half[dir].delay; +} + +int main(int argc, char *argv[]) +{ + int infd, outfd; + common_setup(argv[0]); + setup_locale(); + + opt_register_noarg("--verbose|-v", opt_set_bool, &verbose, + "Print details."); + opt_register_noarg("--help|-h", opt_usage_and_exit, + "[decompress|compress] infile outfile" + "Compress or decompress a gossmap file", + "Print this message."); + + opt_parse(&argc, argv, opt_log_stderr_exit); + if (argc != 4) + opt_usage_and_exit("Needs 4 arguments"); + + infd = open(argv[2], O_RDONLY); + if (infd < 0) + opt_usage_and_exit(tal_fmt(tmpctx, "Cannot open %s for reading: %s", + argv[2], strerror(errno))); + outfd = open(argv[3], O_WRONLY|O_CREAT|O_TRUNC, 0666); + if (outfd < 0) + opt_usage_and_exit(tal_fmt(tmpctx, "Cannot open %s for writing: %s", + argv[3], strerror(errno))); + + if (streq(argv[1], "compress")) { + struct gossmap_node **nodes, *n; + size_t *node_to_compr_idx; + size_t node_count, channel_count; + struct gossmap_chan **chans, *c; + + struct gossmap *gossmap = gossmap_load_fd(tmpctx, infd, NULL, NULL, NULL); + if (!gossmap) + opt_usage_and_exit("Cannot read gossmap"); + + nodes = tal_arr(gossmap, struct gossmap_node *, gossmap_max_node_idx(gossmap)); + for (node_count = 0, n = gossmap_first_node(gossmap); + n; + n = gossmap_next_node(gossmap, n), node_count++) { + nodes[node_count] = n; + } + tal_resize(&nodes, node_count); + if (verbose) + printf("%zu nodes\n", node_count); + + /* nodes with most channels go first */ + asort(nodes, tal_count(nodes), cmp_node_num_chans, NULL); + + /* Create map of gossmap index to compression index */ + node_to_compr_idx = tal_arr(nodes, size_t, gossmap_max_node_idx(gossmap)); + for (size_t i = 0; i < tal_count(nodes); i++) + node_to_compr_idx[gossmap_node_idx(gossmap, nodes[i])] = i; + + if (!write_all(outfd, GC_HEADER, GC_HEADERLEN)) + err(1, "Writing header"); + + /* Now, output channels. First get exact count. */ + for (channel_count = 0, c = gossmap_first_chan(gossmap); + c; + c = gossmap_next_chan(gossmap, c)) { + channel_count++; + } + + if (verbose) + printf("%zu channels\n", channel_count); + chans = tal_arr(gossmap, struct gossmap_chan *, channel_count); + + /* * := {channel_count} {start_nodeidx}*{channel_count} {end_nodeidx}*{channel_count} */ + write_bigsize(outfd, channel_count); + size_t chanidx = 0; + /* We iterate nodes to get to channels. This gives us nicer ordering for compression */ + for (size_t wanted_dir = 0; wanted_dir < 2; wanted_dir++) { + for (n = gossmap_first_node(gossmap); n; n = gossmap_next_node(gossmap, n)) { + for (size_t i = 0; i < n->num_chans; i++) { + int dir; + c = gossmap_nth_chan(gossmap, n, i, &dir); + if (dir != wanted_dir) + continue; + + write_bigsize(outfd, + node_to_compr_idx[gossmap_node_idx(gossmap, n)]); + /* First time reflects channel index for reader */ + if (wanted_dir == 0) + chans[chanidx++] = c; + } + } + } + + /* := * {channel_count*2} */ + /* := {chanidx}*2+{direction} */ + size_t num_disabled = 0; + size_t num_unknown = 0; + for (size_t i = 0; i < channel_count; i++) { + for (size_t dir = 0; dir < 2; dir++) { + if (chans[i]->cupdate_off[dir] == 0) + num_unknown++; + if (!chans[i]->half[dir].enabled) { + write_bigsize(outfd, i * 2 + dir); + num_disabled++; + } + } + } + write_bigsize(outfd, channel_count * 2); + if (verbose) + printf("%zu disabled channels (%zu no update)\n", num_disabled, num_unknown); + + /* := {channel_count}*{capacity_idx} */ + /* := {capacity_count} {capacity_count}*{capacity} */ + u64 *vals = tal_arr(chans, u64, channel_count); + for (size_t i = 0; i < channel_count; i++) { + struct amount_sat sats; + gossmap_chan_get_capacity(gossmap, chans[i], &sats); + vals[i] = sats.satoshis; /* Raw: compression format */ + } + write_template_and_values(outfd, vals, "capacities"); + + /* These are all of same form: one entry per direction per channel */ + /* := {channel_count}*{htlc_min_idx} */ + /* := {htlc_min_count} {htlc_min_count}*{htlc_min} */ + /* := {channel_count}*{htlc_max_idx} */ + /* := {htlc_max_count} {htlc_max_count}*{htlc_max} */ + /* := {channel_count}*{basefee_idx} */ + /* := {basefee_count} {basefee_count}*{basefee} */ + /* := {channel_count}*{propfee_idx} */ + /* := {propfee_count} {propfee_count}*{propfee} */ + /* := {channel_count}*{delay_idx} */ + /* := {delay_count} {delay_count}*{delay} */ + write_bidir_perchan(outfd, gossmap, chans, get_htlc_min, "htlc_min"); + write_bidir_perchan(outfd, gossmap, chans, get_htlc_max, "htlc_max"); + write_bidir_perchan(outfd, gossmap, chans, get_basefee, "basefee"); + write_bidir_perchan(outfd, gossmap, chans, get_propfee, "propfee"); + write_bidir_perchan(outfd, gossmap, chans, get_delay, "delay"); + } else if (streq(argv[1], "decompress")) { + errx(1, "NYI"); + } else + opt_usage_and_exit("Unknown command"); + + common_shutdown(); +}