gossipd: use gossip_query_ex to query only nodes when node probing.

If the peer supports `gossip_query_ex` we can use query_flags to simply
request the node_announcements when probing for nodes, rather than
getting everything.  If a peer doesn't support `gossip_query_ex` then
it's harmless to add it.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

gossipd/queries.c

@@ -49,7 +49,7 @@ static void encoding_add_timestamps(u8 **encoded,
 }
 
 /* Marshal a single query flag (we don't query, so not currently used) */
-static UNNEEDED void encoding_add_query_flag(u8 **encoded, bigsize_t flag)
+static void encoding_add_query_flag(u8 **encoded, bigsize_t flag)
 {
 	towire_bigsize(encoded, flag);
 }
@@ -131,7 +131,7 @@ static bool encoding_end_prepend_type(u8 **encoded, size_t max_bytes)
 }
 
 /* Try compressing, leaving type external */
-static UNNEEDED bool encoding_end_external_type(u8 **encoded, u8 *type, size_t max_bytes)
+static bool encoding_end_external_type(u8 **encoded, u8 *type, size_t max_bytes)
 {
 	if (encoding_end_zlib(encoded, 0))
 		*type = ARR_ZLIB;
@@ -147,10 +147,11 @@ static UNNEEDED bool encoding_end_external_type(u8 **encoded, u8 *type, size_t m
 bool query_short_channel_ids(struct daemon *daemon,
 			     struct peer *peer,
 			     const struct short_channel_id *scids,
+			     const u8 *query_flags,
 			     void (*cb)(struct peer *peer, bool complete))
 {
 	u8 *encoded, *msg;
-
+	struct tlv_query_short_channel_ids_tlvs *tlvs;
 	/* BOLT #7:
 	 *
 	 * 1. type: 261 (`query_short_channel_ids`) (`gossip_queries`)
@@ -160,12 +161,21 @@ bool query_short_channel_ids(struct daemon *daemon,
 	 *     * [`len*byte`:`encoded_short_ids`]
 	 */
 	const size_t reply_overhead = 32 + 2;
-	const size_t max_encoded_bytes = 65535 - 2 - reply_overhead;
+	size_t max_encoded_bytes = 65535 - 2 - reply_overhead;
 
 	/* Can't query if they don't have gossip_queries_feature */
 	if (!peer->gossip_queries_feature)
 		return false;
 
+	/* BOLT #7:
+	 *   - MAY include an optional `query_flags`. If so:
+	 *    - MUST set `encoding_type`, as for `encoded_short_ids`.
+	 *    - Each query flag is a minimally-encoded varint.
+	 *    - MUST encode one query flag per `short_channel_id`.
+	 */
+	if (query_flags)
+		assert(tal_count(query_flags) == tal_count(scids));
+
 	/* BOLT #7:
 	 *
 	 * The sender:
@@ -186,8 +196,30 @@ bool query_short_channel_ids(struct daemon *daemon,
 		return false;
 	}
 
+	if (query_flags) {
+		struct tlv_query_short_channel_ids_tlvs_query_flags *tlvq;
+		tlvs = tlv_query_short_channel_ids_tlvs_new(tmpctx);
+		tlvq = tlvs->query_flags = tal(tlvs,
+			   struct tlv_query_short_channel_ids_tlvs_query_flags);
+ 		tlvq->encoded_query_flags = encoding_start(tlvq);
+		for (size_t i = 0; i < tal_count(query_flags); i++)
+			encoding_add_query_flag(&tlvq->encoded_query_flags,
+						query_flags[i]);
+
+		max_encoded_bytes -= tal_bytelen(encoded);
+		if (!encoding_end_external_type(&tlvq->encoded_query_flags,
+						&tlvq->encoding_type,
+						max_encoded_bytes)) {
+			status_broken("query_short_channel_ids:"
+				      " %zu query_flags is too many",
+				      tal_count(query_flags));
+			return false;
+		}
+	} else
+		tlvs = NULL;
+
 	msg = towire_query_short_channel_ids(NULL, &daemon->chain_hash,
-					     encoded, NULL);
+					     encoded, tlvs);
 	queue_peer_msg(peer, take(msg));
 	peer->scid_query_outstanding = true;
 	peer->scid_query_cb = cb;

gossipd/queries.h

@@ -28,10 +28,11 @@ bool query_channel_range(struct daemon *daemon,
 				    const struct short_channel_id *scids,
 				    bool complete));
 
-/* Ask this peer for info about an array of scids */
+/* Ask this peer for info about an array of scids, with optional query_flags */
 bool query_short_channel_ids(struct daemon *daemon,
 			     struct peer *peer,
 			     const struct short_channel_id *scids,
+			     const u8 *query_flags,
 			     void (*cb)(struct peer *peer, bool complete));
 
 #if DEVELOPER

gossipd/seeker.c

@@ -3,6 +3,7 @@
 #include <ccan/list/list.h>
 #include <ccan/mem/mem.h>
 #include <ccan/tal/tal.h>
+#include <common/decode_array.h>
 #include <common/memleak.h>
 #include <common/pseudorand.h>
 #include <common/status.h>
@@ -78,6 +79,7 @@ struct seeker {
 
 	/* Array of scids for node announcements. */
 	struct short_channel_id *nannounce_scids;
+	u8 *nannounce_query_flags;
 	size_t nannounce_offset;
 };
 
@@ -275,7 +277,7 @@ static void seek_any_unknown_scids(struct seeker *seeker)
 	selected_peer(seeker, peer);
 
 	scids = unknown_scids_arr(tmpctx, seeker);
-	if (!query_short_channel_ids(seeker->daemon, peer, scids,
+	if (!query_short_channel_ids(seeker->daemon, peer, scids, NULL,
 				     scid_query_done))
 		status_failed(STATUS_FAIL_INTERNAL_ERROR,
 			      "seeker: quering %zu scids is too many?",
@@ -340,36 +342,52 @@ static bool next_block_range(struct seeker *seeker,
 	return false;
 }
 
-static struct short_channel_id *get_unannounced_nodes(const tal_t *ctx,
+static bool get_unannounced_nodes(const tal_t *ctx,
 				  struct routing_state *rstate,
 				  size_t off,
-						      size_t max)
+				  size_t max,
+				  struct short_channel_id **scids,
+				  u8 **query_flags)
 {
-	struct short_channel_id *scids;
 	struct node_map_iter it;
 	size_t i = 0, num = 0;
 	struct node *n;
 
 	/* Pick an example short_channel_id at random to query.  As a
 	 * side-effect this gets the node */
-	scids = tal_arr(ctx, struct short_channel_id, max);
+	*scids = tal_arr(ctx, struct short_channel_id, max);
+	*query_flags = tal_arr(ctx, u8, max);
 
 	for (n = node_map_first(rstate->nodes, &it);
 	     n && num < max;
 	     n = node_map_next(rstate->nodes, &it)) {
-		struct chan_map_iter cit;
 		if (n->bcast.index)
 			continue;
 
 		if (i >= off) {
-			scids[num] = first_chan(n, &cit)->scid;
+			struct chan_map_iter cit;
+			struct chan *c = first_chan(n, &cit);
+
+			(*scids)[num] = c->scid;
+			if (c->nodes[0] == n)
+				(*query_flags)[num] = SCID_QF_NODE1;
+			else
+				(*query_flags)[num] = SCID_QF_NODE2;
 			num++;
 		}
 		i++;
 	}
-	if (num < max)
+
-		tal_resize(&scids, num);
+	if (num == 0) {
-	return scids;
+		*scids = tal_free(*scids);
+		*query_flags = tal_free(*query_flags);
+		return false;
+	}
+	if (num < max) {
+		tal_resize(scids, num);
+		tal_resize(query_flags, i - off);
+	}
+	return true;
 }
 
 /* Mutual recursion */
@@ -391,8 +409,11 @@ static void nodeannounce_query_done(struct peer *peer, bool complete)
 		/* Could have closed since we asked. */
 		if (!c)
 			continue;
-		/* We wouldn't have asked if it has both node_announcements */
+		if ((seeker->nannounce_query_flags[i] & SCID_QF_NODE1)
-		if (c->nodes[0]->bcast.index && c->nodes[1]->bcast.index)
+		    && c->nodes[0]->bcast.index)
+			new_nannounce++;
+		if ((seeker->nannounce_query_flags[i] & SCID_QF_NODE2)
+		    && c->nodes[1]->bcast.index)
 			new_nannounce++;
 	}
 
@@ -400,6 +421,7 @@ static void nodeannounce_query_done(struct peer *peer, bool complete)
 		     new_nannounce, num_scids);
 
 	seeker->nannounce_scids = tal_free(seeker->nannounce_scids);
+	seeker->nannounce_query_flags = tal_free(seeker->nannounce_query_flags);
 	seeker->nannounce_offset += num_scids;
 
 	if (!new_nannounce) {
@@ -410,16 +432,15 @@ static void nodeannounce_query_done(struct peer *peer, bool complete)
 	/* Double every time.  We may skip a few, of course, since map
 	 * is changing. */
 	num_scids *= 2;
-	if (num_scids > 8000)
+	/* Don't try to create a query larger than 64k */
-		num_scids = 8000;
+	if (num_scids > 7000)
-
+		num_scids = 7000;
-	seeker->nannounce_scids
-		= get_unannounced_nodes(seeker, seeker->daemon->rstate,
-					seeker->nannounce_offset, num_scids);
 
+	if (!get_unannounced_nodes(seeker, seeker->daemon->rstate,
+				   seeker->nannounce_offset, num_scids,
+				   &seeker->nannounce_scids,
+				   &seeker->nannounce_query_flags)) {
 		/* Nothing unknown at all?  Great, we're done */
-	if (tal_count(seeker->nannounce_scids) == 0) {
-		seeker->nannounce_scids = tal_free(seeker->nannounce_scids);
 		set_state(seeker, NORMAL);
 		return;
 	}
@@ -438,16 +459,10 @@ static void peer_gossip_probe_nannounces(struct seeker *seeker)
 		return;
 	selected_peer(seeker, peer);
 
-	/* Nothing unknown at all?  Great, we're done */
-	if (tal_count(seeker->nannounce_scids) == 0) {
-		seeker->nannounce_scids = tal_free(seeker->nannounce_scids);
-		set_state(seeker, NORMAL);
-		return;
-	}
-
 	set_state(seeker, PROBING_NANNOUNCES);
 	if (!query_short_channel_ids(seeker->daemon, peer,
 				     seeker->nannounce_scids,
+				     seeker->nannounce_query_flags,
 				     nodeannounce_query_done))
 		status_failed(STATUS_FAIL_INTERNAL_ERROR,
 			      "seeker: quering %zu scids is too many?",
@@ -493,13 +508,19 @@ static void process_scid_probe(struct peer *peer,
 	}
 
 	/* Channel probe finished, try asking for 32 unannounced nodes. */
-	seeker->nannounce_offset = 0;
-	seeker->nannounce_scids
-		= get_unannounced_nodes(seeker, seeker->daemon->rstate, 0, 32);
-
 	set_state(seeker, PROBING_NANNOUNCES_NEED_PEER);
-	peer_gossip_probe_nannounces(seeker);
+	seeker->nannounce_offset = 0;
+
+	if (!get_unannounced_nodes(seeker, seeker->daemon->rstate,
+				   seeker->nannounce_offset, 32,
+				   &seeker->nannounce_scids,
+				   &seeker->nannounce_query_flags)) {
+		/* No unknown nodes.  Great! */
+		set_state(seeker, NORMAL);
 		return;
+	}
+
+	peer_gossip_probe_nannounces(seeker);
 }
 
 /* Pick a peer, ask it for a few scids, to check. */

gossipd/test/run-next_block_range.c

@@ -32,6 +32,7 @@ bool query_channel_range(struct daemon *daemon UNNEEDED,
 bool query_short_channel_ids(struct daemon *daemon UNNEEDED,
 			     struct peer *peer UNNEEDED,
 			     const struct short_channel_id *scids UNNEEDED,
+			     const u8 *query_flags UNNEEDED,
 			     void (*cb)(struct peer *peer UNNEEDED, bool complete))
 { fprintf(stderr, "query_short_channel_ids called!\n"); abort(); }
 /* Generated stub for queue_peer_msg */