gossipd: support and use zlib encoding in short_channel_id encoding.

We still use uncompressed if zlib turns out to be larger.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

contrib/Dockerfile.builder

@@ -32,7 +32,8 @@ RUN apt-get -qq update && \
 	python3-setuptools \
 	python-pkg-resources \
 	shellcheck \
-	wget && \
+	wget \
+	zlib1g-dev && \
 	rm -rf /var/lib/apt/lists/*
 
 ENV LANGUAGE=en_US.UTF-8

contrib/Dockerfile.builder.fedora

@@ -17,7 +17,8 @@ RUN dnf update -y && \
 		python3-setuptools \
 		net-tools \
 		valgrind \
-		wget && \
+		wget \
+		zlib-devel && \
 	dnf clean all
 
 RUN wget https://bitcoin.org/bin/bitcoin-core-$BITCOIN_VERSION/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz -O bitcoin.tar.gz && \

contrib/Dockerfile.builder.i386

@@ -32,7 +32,8 @@ RUN apt-get -qq update && \
 	python3-setuptools \
 	python-pkg-resources \
 	shellcheck \
-	wget && \
+	wget \
+	zlib1g-dev && \
 	rm -rf /var/lib/apt/lists/*
 
 ENV LANGUAGE=en_US.UTF-8

gossipd/gossip.c

@@ -56,6 +56,7 @@
 #include <wire/gen_peer_wire.h>
 #include <wire/wire_io.h>
 #include <wire/wire_sync.h>
+#include <zlib.h>
 
 #define GOSSIP_MAX_REACH_ATTEMPTS 10
 
@@ -64,6 +65,18 @@
 #define INITIAL_WAIT_SECONDS	1
 #define MAX_WAIT_SECONDS	300
 
+/* BOLT #7:
+ *
+ * Encoding types:
+ * * `0`: uncompressed array of `short_channel_id` types, in ascending order.
+ * * `1`: array of `short_channel_id` types, in ascending order, compressed with
+ *        zlib<sup>[1](#reference-1)</sup>
+ */
+enum scid_encode_types {
+	SHORTIDS_UNCOMPRESSED = 0,
+	SHORTIDS_ZLIB = 1
+};
+
 /* We put everything in this struct (redundantly) to pass it to timer cb */
 struct important_peerid {
 	struct daemon *daemon;
@@ -424,14 +437,8 @@ static void reached_peer(struct peer *peer, struct io_conn *conn)
 
 static u8 *encode_short_channel_ids_start(const tal_t *ctx)
 {
-	/* BOLT #7:
-	 *
-	 * Encoding types:
-	 * * `0`: uncompressed array of `short_channel_id` types, in ascending
-	 *      order.
-	 */
 	u8 *encoded = tal_arr(tmpctx, u8, 0);
-	towire_u8(&encoded, 0);
+	towire_u8(&encoded, SHORTIDS_ZLIB);
 	return encoded;
 }
 
@@ -441,8 +448,48 @@ static void encode_add_short_channel_id(u8 **encoded,
 	towire_short_channel_id(encoded, scid);
 }
 
+static u8 *zencode_scids(const tal_t *ctx, const u8 *scids, size_t len)
+{
+	u8 *z;
+	int err;
+	unsigned long compressed_len = len;
+
+	/* Prefer to fail if zlib makes it larger */
+	z = tal_arr(ctx, u8, len);
+	err = compress2(z, &compressed_len, scids, len, Z_BEST_COMPRESSION);
+	if (err == Z_OK) {
+		status_trace("short_ids compressed %zu into %lu",
+			     len, compressed_len);
+		tal_resize(&z, compressed_len);
+		return z;
+	}
+	status_trace("short_ids compress %zu returned %i:"
+		     " not compresssing", len, err);
+	return NULL;
+}
+
 static bool encode_short_channel_ids_end(u8 **encoded, size_t max_bytes)
 {
+	u8 *z;
+
+	switch ((enum scid_encode_types)(*encoded)[0]) {
+	case SHORTIDS_ZLIB:
+		z = zencode_scids(tmpctx, *encoded + 1, tal_len(*encoded) - 1);
+		if (z) {
+			tal_resize(encoded, 1 + tal_len(z));
+			memcpy((*encoded) + 1, z, tal_len(z));
+			goto check_length;
+		}
+		(*encoded)[0] = SHORTIDS_UNCOMPRESSED;
+		/* Fall thru */
+	case SHORTIDS_UNCOMPRESSED:
+		goto check_length;
+	}
+
+	status_failed(STATUS_FAIL_INTERNAL_ERROR,
+		      "Unknown short_ids encoding %u", (*encoded)[0]);
+
+check_length:
 #if DEVELOPER
 	if (tal_len(*encoded) > max_scids_encode_bytes)
 		return false;
@@ -792,44 +839,67 @@ static u8 *handle_gossip_msg(struct daemon *daemon, const u8 *msg,
 	return NULL;
 }
 
-/* BOLT #7:
- *
- * the first byte indicates the encoding, the rest contains the data.
- *
- * Encoding types:
- * * `0`: uncompressed array of `short_channel_id` types, in ascending order.
- */
+static u8 *unzlib(const tal_t *ctx, const u8 *encoded, size_t len)
+{
+	/* http://www.zlib.net/zlib_tech.html gives 1032:1 as worst-case,
+	 * which is 67632120 bytes for us.  But they're not encoding zeroes,
+	 * and each scid must be unique.  So 1MB is far more reasonable. */
+	unsigned long unclen = 1024*1024;
+	int zerr;
+	u8 *unc = tal_arr(ctx, u8, unclen);
+
+	zerr = uncompress(unc, &unclen, encoded, len);
+	if (zerr != Z_OK) {
+		status_trace("unzlib: error %i", zerr);
+		return tal_free(unc);
+	}
+
+	/* Truncate and return. */
+	tal_resize(&unc, unclen);
+	return unc;
+}
+
 static struct short_channel_id *decode_short_ids(const tal_t *ctx,
 						 const u8 *encoded)
 {
 	struct short_channel_id *scids;
 	size_t max = tal_len(encoded), n;
-	u8 type;
+	enum scid_encode_types type;
 
 	/* BOLT #7:
 	 *
 	 * The receiver:
-	 *   - if the first byte of `encoded_short_ids` is not zero:
+	 *   - if the first byte of `encoded_short_ids` is not a known encoding
+	 *     type:
 	 *     - MAY fail the connection
 	 *   - if `encoded_short_ids` does not decode into a whole number of
 	 *     `short_channel_id`:
 	 *     - MAY fail the connection
 	 */
 	type = fromwire_u8(&encoded, &max);
-	if (type != 0)
-		return NULL;
+	switch (type) {
+	case SHORTIDS_ZLIB:
+		encoded = unzlib(tmpctx, encoded, max);
+		if (!encoded)
+			return NULL;
+		status_trace("Uncompressed %zu into %zu bytes (%s)",
+			     max, tal_len(encoded), tal_hex(tmpctx, encoded));
+		max = tal_len(encoded);
+		/* fall thru */
+	case SHORTIDS_UNCOMPRESSED:
+		n = 0;
+		scids = tal_arr(ctx, struct short_channel_id, n);
+		while (max) {
+			tal_resize(&scids, n+1);
+			fromwire_short_channel_id(&encoded, &max, &scids[n++]);
+		}
 
-	n = 0;
-	scids = tal_arr(ctx, struct short_channel_id, n);
-	while (max) {
-		tal_resize(&scids, n+1);
-		fromwire_short_channel_id(&encoded, &max, &scids[n++]);
+		/* encoded is set to NULL if we ran over */
+		if (!encoded)
+			return tal_free(scids);
+		return scids;
 	}
-
-	/* encoded is set to NULL if we ran over */
-	if (!encoded)
-		return tal_free(scids);
-	return scids;
+	return NULL;
 }
 
 static void handle_query_short_channel_ids(struct peer *peer, u8 *msg)