From 5735f71e3c1a5c23624c73c9afb4bea26eaf44e2 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 18 May 2022 10:20:16 +0930 Subject: [PATCH] gossipd: don't ever use zlib compression on gossip. This was eliminated this morning in the latest spec. We still accept them, we just don't produce them any more. Changelog-Removed: Protocol: We no longer create gossip messages which use zlib encoding (we still understand them, for now!) Signed-off-by: Rusty Russell --- gossipd/queries.c | 62 +---------- gossipd/test/run-extended-info.c | 175 +++++-------------------------- tests/test_gossip.py | 4 +- 3 files changed, 30 insertions(+), 211 deletions(-) diff --git a/gossipd/queries.c b/gossipd/queries.c index de7d02fd6..9cb8241b0 100644 --- a/gossipd/queries.c +++ b/gossipd/queries.c @@ -52,52 +52,6 @@ static void encoding_add_query_flag(u8 **encoded, bigsize_t flag) towire_bigsize(encoded, flag); } -/* Greg Maxwell asked me privately about using zlib for communicating a set, - * and suggested that we'd be better off using Golomb-Rice coding a-la BIP - * 158. However, naively using Rice encoding isn't a win: we have to get - * more complex and use separate streams. The upside is that it's between - * 2 and 5 times smaller (assuming optimal Rice encoding + gzip). We can add - * that later. */ -static u8 *zencode(const tal_t *ctx, const u8 *scids, size_t len) -{ - u8 *z; - int err; - unsigned long compressed_len = len; - -#ifdef ZLIB_EVEN_IF_EXPANDS - /* Needed for test vectors */ - compressed_len = 128 * 1024; -#endif - /* Prefer to fail if zlib makes it larger */ - z = tal_arr(ctx, u8, compressed_len); - err = compress2(z, &compressed_len, scids, len, Z_DEFAULT_COMPRESSION); - if (err == Z_OK) { - tal_resize(&z, compressed_len); - return z; - } - return NULL; -} - -/* Try compressing *encoded: fails if result would be longer. - * @off is offset to place result in *encoded. - */ -static bool encoding_end_zlib(u8 **encoded, size_t off) -{ - u8 *z; - size_t len = tal_count(*encoded); - - z = zencode(tmpctx, *encoded, len); - if (!z) - return false; - - /* Successful: copy over and trim */ - tal_resize(encoded, off + tal_count(z)); - memcpy(*encoded + off, z, tal_count(z)); - - tal_free(z); - return true; -} - static void encoding_end_no_compress(u8 **encoded, size_t off) { size_t len = tal_count(*encoded); @@ -110,12 +64,8 @@ static void encoding_end_no_compress(u8 **encoded, size_t off) * Prepends encoding type to @encoding. */ static bool encoding_end_prepend_type(u8 **encoded, size_t max_bytes) { - if (encoding_end_zlib(encoded, 1)) - **encoded = ARR_ZLIB; - else { - encoding_end_no_compress(encoded, 1); - **encoded = ARR_UNCOMPRESSED; - } + encoding_end_no_compress(encoded, 1); + **encoded = ARR_UNCOMPRESSED; #if DEVELOPER if (tal_count(*encoded) > dev_max_encoding_bytes) @@ -127,12 +77,8 @@ static bool encoding_end_prepend_type(u8 **encoded, size_t max_bytes) /* Try compressing, leaving type external */ static bool encoding_end_external_type(u8 **encoded, u8 *type, size_t max_bytes) { - if (encoding_end_zlib(encoded, 0)) - *type = ARR_ZLIB; - else { - encoding_end_no_compress(encoded, 0); - *type = ARR_UNCOMPRESSED; - } + encoding_end_no_compress(encoded, 0); + *type = ARR_UNCOMPRESSED; return tal_count(*encoded) <= max_bytes; } diff --git a/gossipd/test/run-extended-info.c b/gossipd/test/run-extended-info.c index 83debe264..845525a26 100644 --- a/gossipd/test/run-extended-info.c +++ b/gossipd/test/run-extended-info.c @@ -1,7 +1,5 @@ #include "config.h" -#define ZLIB_EVEN_IF_EXPANDS 1 - #include "../queries.c" #include #include @@ -112,7 +110,8 @@ void status_fmt(enum log_level level UNNEEDED, { } -static const char *test_vectors[] = { +/* These we can reproduce */ +static const char *test_vectors_nozlib[] = { "{\n" " \"msg\" : {\n" " \"type\" : \"QueryChannelRange\",\n" @@ -137,34 +136,6 @@ static const char *test_vectors[] = { " \"msg\" : {\n" " \"type\" : \"ReplyChannelRange\",\n" " \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n" - " \"firstBlockNum\" : 756230,\n" - " \"numberOfBlocks\" : 1500,\n" - " \"complete\" : 1,\n" - " \"shortChannelIds\" : {\n" - " \"encoding\" : \"UNCOMPRESSED\",\n" - " \"array\" : [ \"0x0x142\", \"0x0x15465\", \"0x69x42692\" ]\n" - " }\n" - " },\n" - " \"hex\" : \"01080f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206000b8a06000005dc01001900000000000000008e0000000000003c69000000000045a6c4\"\n" - "}\n", - "{\n" - " \"msg\" : {\n" - " \"type\" : \"ReplyChannelRange\",\n" - " \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n" - " \"firstBlockNum\" : 1600,\n" - " \"numberOfBlocks\" : 110,\n" - " \"complete\" : 1,\n" - " \"shortChannelIds\" : {\n" - " \"encoding\" : \"COMPRESSED_ZLIB\",\n" - " \"array\" : [ \"0x0x142\", \"0x0x15465\", \"0x4x3318\" ]\n" - " }\n" - " },\n" - " \"hex\" : \"01080f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206000006400000006e01001601789c636000833e08659309a65878be010010a9023a\"\n" - "}\n", - "{\n" - " \"msg\" : {\n" - " \"type\" : \"ReplyChannelRange\",\n" - " \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n" " \"firstBlockNum\" : 122334,\n" " \"numberOfBlocks\" : 1500,\n" " \"complete\" : 1,\n" @@ -202,45 +173,6 @@ static const char *test_vectors[] = { "}\n", "{\n" " \"msg\" : {\n" - " \"type\" : \"ReplyChannelRange\",\n" - " \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n" - " \"firstBlockNum\" : 122334,\n" - " \"numberOfBlocks\" : 1500,\n" - " \"complete\" : 1,\n" - " \"shortChannelIds\" : {\n" - " \"encoding\" : \"COMPRESSED_ZLIB\",\n" - " \"array\" : [ \"0x0x12355\", \"0x7x30934\", \"0x70x57793\" ]\n" - " },\n" - " \"timestamps\" : {\n" - " \"encoding\" : \"COMPRESSED_ZLIB\",\n" - " \"timestamps\" : [ {\n" - " \"timestamp1\" : 164545,\n" - " \"timestamp2\" : 948165\n" - " }, {\n" - " \"timestamp1\" : 489645,\n" - " \"timestamp2\" : 4786864\n" - " }, {\n" - " \"timestamp1\" : 46456,\n" - " \"timestamp2\" : 9788415\n" - " } ]\n" - " },\n" - " \"checksums\" : {\n" - " \"checksums\" : [ {\n" - " \"checksum1\" : 1111,\n" - " \"checksum2\" : 2222\n" - " }, {\n" - " \"checksum1\" : 3333,\n" - " \"checksum2\" : 4444\n" - " }, {\n" - " \"checksum1\" : 5555,\n" - " \"checksum2\" : 6666\n" - " } ]\n" - " }\n" - " },\n" - " \"hex\" : \"01080f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e22060001ddde000005dc01001801789c63600001036730c55e710d4cbb3d3c080017c303b1012201789c63606a3ac8c0577e9481bd622d8327d7060686ad150c53a3ff0300554707db031800000457000008ae00000d050000115c000015b300001a0a\"\n" - "}\n", - "{\n" - " \"msg\" : {\n" " \"type\" : \"QueryShortChannelIds\",\n" " \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n" " \"shortChannelIds\" : {\n" @@ -251,48 +183,6 @@ static const char *test_vectors[] = { " },\n" " \"hex\" : \"01050f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206001900000000000000008e0000000000003c69000000000045a6c4\"\n" "}\n", - "{\n" - " \"msg\" : {\n" - " \"type\" : \"QueryShortChannelIds\",\n" - " \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n" - " \"shortChannelIds\" : {\n" - " \"encoding\" : \"COMPRESSED_ZLIB\",\n" - " \"array\" : [ \"0x0x4564\", \"0x2x47550\", \"0x69x42692\" ]\n" - " },\n" - " \"extensions\" : [ ]\n" - " },\n" - " \"hex\" : \"01050f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206001801789c63600001c12b608a69e73e30edbaec0800203b040e\"\n" - "}\n", - "{\n" - " \"msg\" : {\n" - " \"type\" : \"QueryShortChannelIds\",\n" - " \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n" - " \"shortChannelIds\" : {\n" - " \"encoding\" : \"UNCOMPRESSED\",\n" - " \"array\" : [ \"0x0x12232\", \"0x0x15556\", \"0x69x42692\" ]\n" - " },\n" - " \"extensions\" : [ {\n" - " \"encoding\" : \"COMPRESSED_ZLIB\",\n" - " \"array\" : [ 1, 2, 4 ]\n" - " } ]\n" - " },\n" - " \"hex\" : \"01050f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e22060019000000000000002fc80000000000003cc4000000000045a6c4010c01789c6364620100000e0008\"\n" - "}\n", - "{\n" - " \"msg\" : {\n" - " \"type\" : \"QueryShortChannelIds\",\n" - " \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n" - " \"shortChannelIds\" : {\n" - " \"encoding\" : \"COMPRESSED_ZLIB\",\n" - " \"array\" : [ \"0x0x14200\", \"0x0x46645\", \"0x69x42692\" ]\n" - " },\n" - " \"extensions\" : [ {\n" - " \"encoding\" : \"COMPRESSED_ZLIB\",\n" - " \"array\" : [ 1, 2, 4 ]\n" - " } ]\n" - " },\n" - " \"hex\" : \"01050f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206001801789c63600001f30a30c5b0cd144cb92e3b020017c6034a010c01789c6364620100000e0008\"\n" - "}\n", }; static void get_chainhash(const char *test_vector, @@ -323,14 +213,9 @@ static u8 *get_scid_array(const tal_t *ctx, assert(json_to_short_channel_id(test_vector, t, &scid)); encoding_add_short_channel_id(&encoded, &scid); } - if (json_tok_streq(test_vector, encoding, "UNCOMPRESSED")) { - encoding_end_no_compress(&encoded, 1); - encoded[0] = ARR_UNCOMPRESSED; - } else { - assert(json_tok_streq(test_vector, encoding, "COMPRESSED_ZLIB")); - assert(encoding_end_zlib(&encoded, 1)); - encoded[0] = ARR_ZLIB; - } + assert(json_tok_streq(test_vector, encoding, "UNCOMPRESSED")); + encoding_end_no_compress(&encoded, 1); + encoded[0] = ARR_UNCOMPRESSED; return encoded; } @@ -404,17 +289,10 @@ static u8 *test_reply_channel_range(const char *test_vector, const jsmntok_t *ob &ts.timestamp_node_id_2)); encoding_add_timestamps(&tlvs->timestamps_tlv->encoded_timestamps, &ts); } - if (json_tok_streq(test_vector, encodingtok, "UNCOMPRESSED")) { - encoding_end_no_compress(&tlvs->timestamps_tlv->encoded_timestamps, - 0); - tlvs->timestamps_tlv->encoding_type = ARR_UNCOMPRESSED; - } else { - assert(json_tok_streq(test_vector, encodingtok, - "COMPRESSED_ZLIB")); - assert(encoding_end_zlib(&tlvs->timestamps_tlv->encoded_timestamps, - 0)); - tlvs->timestamps_tlv->encoding_type = ARR_ZLIB; - } + assert(json_tok_streq(test_vector, encodingtok, "UNCOMPRESSED")); + encoding_end_no_compress(&tlvs->timestamps_tlv->encoded_timestamps, + 0); + tlvs->timestamps_tlv->encoding_type = ARR_UNCOMPRESSED; } opt = json_get_member(test_vector, obj, "checksums"); @@ -464,14 +342,9 @@ get_query_flags_array(const tal_t *ctx, assert(json_to_u64(test_vector, t, &f)); encoding_add_query_flag(&tlv->encoded_query_flags, f); } - if (json_tok_streq(test_vector, encoding, "UNCOMPRESSED")) { - encoding_end_no_compress(&tlv->encoded_query_flags, 0); - tlv->encoding_type = ARR_UNCOMPRESSED; - } else { - assert(json_tok_streq(test_vector, encoding, "COMPRESSED_ZLIB")); - assert(encoding_end_zlib(&tlv->encoded_query_flags, 0)); - tlv->encoding_type = ARR_ZLIB; - } + assert(json_tok_streq(test_vector, encoding, "UNCOMPRESSED")); + encoding_end_no_compress(&tlv->encoded_query_flags, 0); + tlv->encoding_type = ARR_UNCOMPRESSED; return tlv; } @@ -509,7 +382,7 @@ int main(int argc, char *argv[]) common_setup(argv[0]); - for (size_t i = 0; i < ARRAY_SIZE(test_vectors); i++) { + for (size_t i = 0; i < ARRAY_SIZE(test_vectors_nozlib); i++) { jsmn_parser parser; u8 *m; const char *hex_m; @@ -517,22 +390,22 @@ int main(int argc, char *argv[]) jsmn_init(&parser); assert(jsmn_parse(&parser, - test_vectors[i], strlen(test_vectors[i]), + test_vectors_nozlib[i], strlen(test_vectors_nozlib[i]), toks, tal_count(toks)) > 0); - msg = json_get_member(test_vectors[i], toks, "msg"); - hex = json_get_member(test_vectors[i], toks, "hex"); - type = json_get_member(test_vectors[i], msg, "type"); - if (json_tok_streq(test_vectors[i], type, "QueryChannelRange")) - m = test_query_channel_range(test_vectors[i], msg); - else if (json_tok_streq(test_vectors[i], type, "ReplyChannelRange")) - m = test_reply_channel_range(test_vectors[i], msg); - else if (json_tok_streq(test_vectors[i], type, "QueryShortChannelIds")) - m = test_query_short_channel_ids(test_vectors[i], msg); + msg = json_get_member(test_vectors_nozlib[i], toks, "msg"); + hex = json_get_member(test_vectors_nozlib[i], toks, "hex"); + type = json_get_member(test_vectors_nozlib[i], msg, "type"); + if (json_tok_streq(test_vectors_nozlib[i], type, "QueryChannelRange")) + m = test_query_channel_range(test_vectors_nozlib[i], msg); + else if (json_tok_streq(test_vectors_nozlib[i], type, "ReplyChannelRange")) + m = test_reply_channel_range(test_vectors_nozlib[i], msg); + else if (json_tok_streq(test_vectors_nozlib[i], type, "QueryShortChannelIds")) + m = test_query_short_channel_ids(test_vectors_nozlib[i], msg); else abort(); hex_m = tal_hex(m, m); - assert(json_tok_streq(test_vectors[i], hex, hex_m)); + assert(json_tok_streq(test_vectors_nozlib[i], hex, hex_m)); tal_free(m); } tal_free(toks); diff --git a/tests/test_gossip.py b/tests/test_gossip.py index e5d76d6af..0b1cd7619 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -872,7 +872,7 @@ def test_gossip_query_channel_range(node_factory, bitcoind, chainparams): filters=['0109', '0012']) assert len(msgs) == 2 - # This should actually be large enough for zlib to kick in! + # This used to be large enough for zlib to kick in, but no longer! scid34, _ = l3.fundchannel(l4, 10**5) mine_funding_to_announce(bitcoind, [l1, l2, l3, l4]) l2.daemon.wait_for_log('Received node_announcement for node ' + l4.info['id']) @@ -886,7 +886,7 @@ def test_gossip_query_channel_range(node_factory, bitcoind, chainparams): genesis_blockhash, 0, 65535, filters=['0109', '0012']) - encoded = subprocess.run(['devtools/mkencoded', '--scids', '01', scid12, scid23, scid34], + encoded = subprocess.run(['devtools/mkencoded', '--scids', '00', scid12, scid23, scid34], check=True, timeout=TIMEOUT, stdout=subprocess.PIPE).stdout.strip().decode()