diff --git a/contrib/Dockerfile.builder b/contrib/Dockerfile.builder index ba1f901a9..f3821e6e7 100644 --- a/contrib/Dockerfile.builder +++ b/contrib/Dockerfile.builder @@ -32,7 +32,8 @@ RUN apt-get -qq update && \ python3-setuptools \ python-pkg-resources \ shellcheck \ - wget && \ + wget \ + zlib1g-dev && \ rm -rf /var/lib/apt/lists/* ENV LANGUAGE=en_US.UTF-8 diff --git a/contrib/Dockerfile.builder.fedora b/contrib/Dockerfile.builder.fedora index 91ab2c71c..517e4da34 100644 --- a/contrib/Dockerfile.builder.fedora +++ b/contrib/Dockerfile.builder.fedora @@ -17,7 +17,8 @@ RUN dnf update -y && \ python3-setuptools \ net-tools \ valgrind \ - wget && \ + wget \ + zlib-devel && \ dnf clean all RUN wget https://bitcoin.org/bin/bitcoin-core-$BITCOIN_VERSION/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz -O bitcoin.tar.gz && \ diff --git a/contrib/Dockerfile.builder.i386 b/contrib/Dockerfile.builder.i386 index b425b47b8..c1421c113 100644 --- a/contrib/Dockerfile.builder.i386 +++ b/contrib/Dockerfile.builder.i386 @@ -32,7 +32,8 @@ RUN apt-get -qq update && \ python3-setuptools \ python-pkg-resources \ shellcheck \ - wget && \ + wget \ + zlib1g-dev && \ rm -rf /var/lib/apt/lists/* ENV LANGUAGE=en_US.UTF-8 diff --git a/gossipd/gossip.c b/gossipd/gossip.c index dfeee3d43..44072c9dc 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -56,6 +56,7 @@ #include #include #include +#include #define GOSSIP_MAX_REACH_ATTEMPTS 10 @@ -64,6 +65,18 @@ #define INITIAL_WAIT_SECONDS 1 #define MAX_WAIT_SECONDS 300 +/* BOLT #7: + * + * Encoding types: + * * `0`: uncompressed array of `short_channel_id` types, in ascending order. + * * `1`: array of `short_channel_id` types, in ascending order, compressed with + * zlib[1](#reference-1) + */ +enum scid_encode_types { + SHORTIDS_UNCOMPRESSED = 0, + SHORTIDS_ZLIB = 1 +}; + /* We put everything in this struct (redundantly) to pass it to timer cb */ struct important_peerid { struct daemon *daemon; @@ -424,14 +437,8 @@ static void reached_peer(struct peer *peer, struct io_conn *conn) static u8 *encode_short_channel_ids_start(const tal_t *ctx) { - /* BOLT #7: - * - * Encoding types: - * * `0`: uncompressed array of `short_channel_id` types, in ascending - * order. - */ u8 *encoded = tal_arr(tmpctx, u8, 0); - towire_u8(&encoded, 0); + towire_u8(&encoded, SHORTIDS_ZLIB); return encoded; } @@ -441,8 +448,48 @@ static void encode_add_short_channel_id(u8 **encoded, towire_short_channel_id(encoded, scid); } +static u8 *zencode_scids(const tal_t *ctx, const u8 *scids, size_t len) +{ + u8 *z; + int err; + unsigned long compressed_len = len; + + /* Prefer to fail if zlib makes it larger */ + z = tal_arr(ctx, u8, len); + err = compress2(z, &compressed_len, scids, len, Z_BEST_COMPRESSION); + if (err == Z_OK) { + status_trace("short_ids compressed %zu into %lu", + len, compressed_len); + tal_resize(&z, compressed_len); + return z; + } + status_trace("short_ids compress %zu returned %i:" + " not compresssing", len, err); + return NULL; +} + static bool encode_short_channel_ids_end(u8 **encoded, size_t max_bytes) { + u8 *z; + + switch ((enum scid_encode_types)(*encoded)[0]) { + case SHORTIDS_ZLIB: + z = zencode_scids(tmpctx, *encoded + 1, tal_len(*encoded) - 1); + if (z) { + tal_resize(encoded, 1 + tal_len(z)); + memcpy((*encoded) + 1, z, tal_len(z)); + goto check_length; + } + (*encoded)[0] = SHORTIDS_UNCOMPRESSED; + /* Fall thru */ + case SHORTIDS_UNCOMPRESSED: + goto check_length; + } + + status_failed(STATUS_FAIL_INTERNAL_ERROR, + "Unknown short_ids encoding %u", (*encoded)[0]); + +check_length: #if DEVELOPER if (tal_len(*encoded) > max_scids_encode_bytes) return false; @@ -792,44 +839,67 @@ static u8 *handle_gossip_msg(struct daemon *daemon, const u8 *msg, return NULL; } -/* BOLT #7: - * - * the first byte indicates the encoding, the rest contains the data. - * - * Encoding types: - * * `0`: uncompressed array of `short_channel_id` types, in ascending order. - */ +static u8 *unzlib(const tal_t *ctx, const u8 *encoded, size_t len) +{ + /* http://www.zlib.net/zlib_tech.html gives 1032:1 as worst-case, + * which is 67632120 bytes for us. But they're not encoding zeroes, + * and each scid must be unique. So 1MB is far more reasonable. */ + unsigned long unclen = 1024*1024; + int zerr; + u8 *unc = tal_arr(ctx, u8, unclen); + + zerr = uncompress(unc, &unclen, encoded, len); + if (zerr != Z_OK) { + status_trace("unzlib: error %i", zerr); + return tal_free(unc); + } + + /* Truncate and return. */ + tal_resize(&unc, unclen); + return unc; +} + static struct short_channel_id *decode_short_ids(const tal_t *ctx, const u8 *encoded) { struct short_channel_id *scids; size_t max = tal_len(encoded), n; - u8 type; + enum scid_encode_types type; /* BOLT #7: * * The receiver: - * - if the first byte of `encoded_short_ids` is not zero: + * - if the first byte of `encoded_short_ids` is not a known encoding + * type: * - MAY fail the connection * - if `encoded_short_ids` does not decode into a whole number of * `short_channel_id`: * - MAY fail the connection */ type = fromwire_u8(&encoded, &max); - if (type != 0) - return NULL; + switch (type) { + case SHORTIDS_ZLIB: + encoded = unzlib(tmpctx, encoded, max); + if (!encoded) + return NULL; + status_trace("Uncompressed %zu into %zu bytes (%s)", + max, tal_len(encoded), tal_hex(tmpctx, encoded)); + max = tal_len(encoded); + /* fall thru */ + case SHORTIDS_UNCOMPRESSED: + n = 0; + scids = tal_arr(ctx, struct short_channel_id, n); + while (max) { + tal_resize(&scids, n+1); + fromwire_short_channel_id(&encoded, &max, &scids[n++]); + } - n = 0; - scids = tal_arr(ctx, struct short_channel_id, n); - while (max) { - tal_resize(&scids, n+1); - fromwire_short_channel_id(&encoded, &max, &scids[n++]); + /* encoded is set to NULL if we ran over */ + if (!encoded) + return tal_free(scids); + return scids; } - - /* encoded is set to NULL if we ran over */ - if (!encoded) - return tal_free(scids); - return scids; + return NULL; } static void handle_query_short_channel_ids(struct peer *peer, u8 *msg)