bolt-gen: fixup the devtool/decodemsg printing facility

Fixup TLV handling in the bolt printing utility, `devtools/decodemsg`

devtools/Makefile

@@ -38,32 +38,30 @@ devtools/gen_print_onion_wire.h: $(DEVTOOL_BOLT_DEPS) wire/gen_onion_wire_csv
 devtools/gen_print_onion_wire.c: $(DEVTOOL_BOLT_DEPS) wire/gen_onion_wire_csv
 	$(BOLT_GEN) -P --page impl ${@:.c=.h} onion_type < wire/gen_onion_wire_csv > $@
 
-devtools/bolt11-cli: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/bolt11-cli.o
+devtools/bolt11-cli: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/tlvstream.o devtools/bolt11-cli.o
 
-devtools/decodemsg: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/decodemsg.o
+devtools/decodemsg: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/tlvstream.o devtools/decodemsg.o
 
-devtools/dump-gossipstore: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/dump-gossipstore.o gossipd/gen_gossip_store.o
+devtools/dump-gossipstore: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/tlvstream.o devtools/dump-gossipstore.o gossipd/gen_gossip_store.o
 
 devtools/dump-gossipstore.o: gossipd/gen_gossip_store.h
 
-devtools/create-gossipstore: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/create-gossipstore.o gossipd/gen_gossip_store.o
+devtools/create-gossipstore: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/tlvstream.o devtools/create-gossipstore.o gossipd/gen_gossip_store.o
 devtools/create-gossipstore.o: gossipd/gen_gossip_store.h
 
 devtools/onion.c: ccan/config.h
 
-devtools/onion: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/onion.o common/sphinx.o
+devtools/onion: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/tlvstream.o devtools/onion.o common/sphinx.o
 
-devtools/gossipwith: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/gen_peer_wire.o devtools/gossipwith.o common/cryptomsg.o common/cryptomsg.o common/crypto_sync.o
+devtools/gossipwith: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/tlvstream.o wire/gen_peer_wire.o devtools/gossipwith.o common/cryptomsg.o common/cryptomsg.o common/crypto_sync.o
 
 $(DEVTOOLS_OBJS) $(DEVTOOLS_TOOL_OBJS): wire/wire.h devtools/gen_print_wire.h devtools/gen_print_onion_wire.h
 devtools/gen_print_wire.o: devtools/gen_print_wire.h wire/gen_peer_wire.h devtools/print_wire.h
 devtools/gen_print_onion_wire.o: devtools/gen_print_onion_wire.h devtools/print_wire.h
 
-devtools/bolt11-cli: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/bolt11-cli.o
+devtools/mkcommit: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) common/derive_basepoints.o common/keyset.o common/key_derive.o common/initial_commit_tx.o common/permute_tx.o wire/fromwire.o wire/towire.o wire/tlvstream.o devtools/mkcommit.o channeld/full_channel.o common/initial_channel.o common/htlc_state.o common/pseudorand.o common/htlc_tx.o channeld/commit_tx.o common/htlc_trim.o
 
-devtools/mkcommit: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) common/derive_basepoints.o common/keyset.o common/key_derive.o common/initial_commit_tx.o common/permute_tx.o wire/fromwire.o wire/towire.o devtools/mkcommit.o channeld/full_channel.o common/initial_channel.o common/htlc_state.o common/pseudorand.o common/htlc_tx.o channeld/commit_tx.o common/htlc_trim.o
+devtools/mkfunding: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/tlvstream.o common/funding_tx.o common/utxo.o common/permute_tx.o common/key_derive.o devtools/mkfunding.o
-
-devtools/mkfunding: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o common/funding_tx.o common/utxo.o common/permute_tx.o common/key_derive.o devtools/mkfunding.o
 
 # Make sure these depend on everything.
 ALL_PROGRAMS += $(DEVTOOLS)

devtools/print_wire.c

@@ -6,6 +6,7 @@
 #include <errno.h>
 #include <inttypes.h>
 #include <stdio.h>
+#include <wire/tlvstream.h>
 
 void printwire_u8(const char *fieldname, const u8 *v)
 {
@@ -161,6 +162,50 @@ void printwire_u8_array(const char *fieldname, const u8 **cursor, size_t *plen,
 	printf("]\n");
 }
 
+static const struct tlv_print_record_type *
+find_print_record_type(u64 type,
+		 const struct tlv_print_record_type types[],
+		 size_t num_types)
+{
+	for (size_t i = 0; i < num_types; i++)
+		if (types[i].type == type)
+			return types + i;
+	return NULL;
+}
+
+void printwire_tlvs(const char *fieldname, const u8 **cursor, size_t *plen,
+		    const struct tlv_print_record_type types[],
+		    size_t num_types)
+{
+	while (*plen > 0) {
+		u64 type, length;
+		const struct tlv_print_record_type *ptype;
+
+		type = fromwire_bigsize(cursor, plen);
+		if (!*cursor)
+			goto fail;
+		length = fromwire_bigsize(cursor, plen);
+		if (!*cursor)
+			goto fail;
+
+		ptype = find_print_record_type(type, types, num_types);
+		if (ptype) {
+			size_t tlvlen = length;
+			printf("{\ntype=%"PRIu64"\nlen=%"PRIu64"\n", type, length);
+			ptype->print(fieldname, cursor, &tlvlen);
+			if (!*cursor)
+				goto fail;
+			printf("}\n");
+			*plen -= length;
+		} else
+			printf("**TYPE #%ld UNKNOWN for TLV %s**\n", type, fieldname);
+	}
+	return;
+
+fail:
+	printf("**TRUNCATED TLV %s**\n", fieldname);
+}
+
 #define PRINTWIRE_TYPE_TO_STRING(T, N)					\
 	void printwire_##N(const char *fieldname, const T *v)		\
 	{								\

devtools/print_wire.h

@@ -5,11 +5,18 @@
 #include <common/wireaddr.h>
 #include <wire/gen_peer_wire.h>
 
+struct tlv_print_record_type {
+	u64 type;
+	void (*print)(const char *tlv_name, const u8 **cursor, size_t *plen);
+};
+
 void printwire_u8(const char *fieldname, const u8 *v);
 void printwire_u16(const char *fieldname, const u16 *v);
 void printwire_u32(const char *fieldname, const u32 *v);
 void printwire_u64(const char *fieldname, const u64 *v);
 void printwire_u8_array(const char *fieldname, const u8 **cursor, size_t *plen, size_t len);
+void printwire_tlvs(const char *tlv_name, const u8 **cursor, size_t *plen,
+		    const struct tlv_print_record_type types[], size_t num_types);
 
 void printwire_bitcoin_blkid(const char *fieldname, const struct bitcoin_blkid *bitcoin_blkid);
 void printwire_bitcoin_txid(const char *fieldname, const struct bitcoin_txid *bitcoin_txid);

tools/gen/impl_template

@@ -148,7 +148,7 @@ fromwire_${type_}(${'ctx, ' if f.needs_context() else ''}&cursor, &plen, ${'*' i
 </%def>
 % for tlv in tlvs.values(): ## START TLV's
 
-struct ${tlv.struct_name()} *${tlv.struct_name()}_new(const tal_t *ctx)
+${tlv.type_name()} *${tlv.struct_name()}_new(const tal_t *ctx)
 {
 	/* Initialize everything to NULL. (Quiet, C pedants!) */
 	return talz(ctx, struct ${tlv.struct_name()});

tools/gen/print_impl_template

@@ -2,6 +2,7 @@
 /* Do not modify this file! Modify the _csv file it was generated from. */
 
 #include "${options.header_filename}"
+#include <ccan/array_size/array_size.h>
 #include <ccan/mem/mem.h>
 #include <ccan/tal/str/str.h>
 #include <common/utils.h>
@@ -22,21 +23,6 @@ void print${options.enum_name}_message(const u8 *msg)
 	printf("UNKNOWN: %s\\n", tal_hex(msg, msg));
 }
 
-void print${options.enum_name}_tlv_message(const char *tlv_name, const u8 *msg)
-{
-% if not bool(tlvs):
-	printf("~~ No TLV definition found for %s ~~\\n", tlv_name);
-% else:
-    % for tlv in tlvs:
-	if (strcmp(tlv_name, "${tlv.name}") == 0) {
-		printwire_${tlv.name}("${tlv.name}", msg);
-		return;
-	}	
-    % endfor
-	printf("ERR: Unknown TLV message type: %s\n", tlv_name);
-% endif
-}
-
 ## 'component' for 'truncate check 
 <%def name="truncate_check(nested=False)">
 	if (!${ '*' if nested else '' }cursor) {
@@ -46,30 +32,41 @@ void print${options.enum_name}_tlv_message(const char *tlv_name, const u8 *msg)
 </%def> \
 ## definition for printing field sets
 <%def name="print_fieldset(fields, nested, cursor, plen)">
-## FIXME: optional field handling omitted since we only generate these for bolts rn
 % for f in fields:
     % if f.len_field_of:
 	${f.type_obj.type_name()} ${f.name} = fromwire_${f.type_obj.name}(${cursor}, ${plen});${truncate_check(nested)} <% continue %> \
     % endif	
 	printf("${f.name}=");
-    % if f.is_array() or f.is_varlen():
+    % if f.type_obj.is_tlv():
+	printwire_tlvs(tal_fmt(NULL, "%s.${f.name}", fieldname), ${cursor}, ${plen}, print_tlvs_${f.type_obj.tlv.name}, ARRAY_SIZE(print_tlvs_${f.type_obj.tlv.name}));
+    % elif f.is_array() or f.is_varlen():
 	% if f.type_obj.has_array_helper():
 	printwire_${f.type_obj.name}_array(tal_fmt(NULL, "%s.${f.name}", fieldname), ${cursor}, ${plen}, ${f.size()});
 	% else:
 	printf("[");
 	for (size_t i = 0; i < ${f.size()}; i++) {
-		${f.type_obj.type_name()} v;
+	    % if f.type_obj.is_subtype():
-	    % if f.type_obj.is_assignable():
+		printwire_${f.type_obj.name}(tal_fmt(NULL, "%s.${f.name}", fieldname), ${cursor}, ${plen});
-		v = fromwire_${f.type_obj.name}(${cursor}, ${plen});
 	    % else:
+		${f.type_obj.type_name()} v;
+		% if f.type_obj.is_assignable():
+		v = fromwire_${f.type_obj.name}(${cursor}, ${plen});
+		% else:
 		fromwire_${f.type_obj.name}(${cursor}, ${plen}, &v);
+		% endif
+		if (!*cursor) {
+			printf("**TRUNCATED**\n");
+			return;
+		}
+<% typename = f.type_obj.name if not f.type_obj.is_truncated() else f.type_obj.name[1:] %>\
+		printwire_${typename}(tal_fmt(NULL, "%s.${f.name}", fieldname), &v);
 	    % endif
-${truncate_check(nested)} \
-		printwire_${f.type_obj.name}(tal_fmt(NULL, "%s.${f.name}", fieldname), &v);
 	}
 	printf("]");
 	% endif
 ${truncate_check(nested)} \
+    % elif f.type_obj.is_subtype():
+	printwire_${f.type_obj.name}(tal_fmt(NULL, "%s.${f.name}", fieldname), ${cursor}, ${plen});
     % else:
 	    % if f.type_obj.is_assignable():
 	${f.type_obj.type_name()} ${f.name} = fromwire_${f.type_obj.name}(${cursor}, ${plen});
@@ -77,20 +74,35 @@ ${truncate_check(nested)} \
 	${f.type_obj.type_name()} ${f.name};
 	fromwire_${f.type_obj.name}(${cursor}, ${plen}, &${f.name});
 	    % endif
-	printwire_${f.type_obj.name}(tal_fmt(NULL, "%s.${f.name}", fieldname), &${f.name}); ${truncate_check(nested)} \
+	<% typename = f.type_obj.name if not f.type_obj.is_truncated() else f.type_obj.name[1:] %>
+	printwire_${typename}(tal_fmt(NULL, "%s.${f.name}", fieldname), &${f.name}); ${truncate_check(nested)} \
     % endif
 % endfor
 </%def> \
-
 ## Definitions for 'subtypes'
 % for subtype in subtypes:
-static void printwire_${subtype.name}(const char *fieldname, const u9 **cursor, size_t *plen)
+
+static void printwire_${subtype.name}(const char *fieldname, const u8 **cursor, size_t *plen)
 {
 	${print_fieldset(subtype.fields.values(), True, 'cursor', 'plen')}
 }
 % endfor
+% for tlv in tlvs.values():
 
-## FIXME: handling for tlv's :/
+    % for msg in tlv.messages.values():
+static void printwire_${msg.struct_name()}(const char *fieldname, const u8 **cursor, size_t *plen)
+{
+	printf("(msg_name=%s)\n", "${msg.name}");
+	${print_fieldset(msg.fields.values(), True, 'cursor', 'plen')}
+}
+    % endfor
+
+static const struct tlv_print_record_type print_tlvs_${tlv.name}[] = {
+    % for msg in tlv.messages.values():
+	{ ${msg.number}, printwire_${msg.struct_name()} },
+    % endfor
+};
+% endfor
 % for msg in messages:
 void printwire_${msg.name}(const char *fieldname, const u8 *cursor)
 {
@@ -107,3 +119,17 @@ ${print_fieldset(msg.fields.values(), False, '&cursor', '&plen')}
 		printf("EXTRA: %s\n", tal_hexstr(NULL, cursor, plen));
 }
 % endfor
+
+void print${options.enum_name}_tlv_message(const char *tlv_name, const u8 *msg) {
+% if bool(tlvs):
+	size_t plen;
+    % for tlv_name in tlvs:
+	plen = tal_count(msg);
+	if (strcmp(tlv_name, "${tlv_name}") == 0) {
+		printwire_tlvs(tlv_name, &msg, &plen, print_tlvs_${tlv_name}, ARRAY_SIZE(print_tlvs_${tlv_name}));
+	}
+    % endfor
+% else:
+	printf("ERR: No TLV definition found for %s\n", tlv_name);
+% endif
+}

tools/generate-bolts.py

@@ -270,6 +270,9 @@ class Type(FieldSet):
     def is_subtype(self):
         return bool(self.fields)
 
+    def is_truncated(self):
+        return self.name in self.truncated_typedefs
+
     def needs_context(self):
         return self.is_varsize() or any([field.needs_context() for field in self.fields.values()])
 
@@ -329,6 +332,9 @@ class Tlv(object):
                                            struct_prefix=self.struct_name(),
                                            comments=comments)
 
+    def type_name(self):
+        return 'struct ' + self.struct_name()
+
     def struct_name(self):
         return "tlv_{}".format(self.name)
 

tools/test/Makefile

@@ -7,8 +7,8 @@
 # and run a test case.
 check-units: check-tools
 
-TOOL_GEN_SRC := tools/test/gen_test.c
+TOOL_GEN_SRC := tools/test/gen_test.c tools/test/gen_print.c
-TOOL_GEN_HEADER := tools/test/gen_test.h
+TOOL_GEN_HEADER := tools/test/gen_test.h tools/test/gen_print.h
 TOOL_TEST_SRC := $(wildcard tools/test/run-*.c)
 TOOL_TEST_OBJS := $(TOOL_TEST_SRC:.c=.o)
 TOOL_TEST_PROGRAMS := $(TOOL_TEST_OBJS:.o=)
@@ -18,9 +18,9 @@ TOOL_TEST_COMMON_OBJS :=		\
 
 TOOLS_WIRE_DEPS := $(BOLT_DEPS) tools/test/test_cases $(wildcard tools/gen/*_template)
 
-$(TOOL_TEST_SRC): $(TOOL_GEN_HEADER)
+$(TOOL_TEST_SRC) $(TOOL_GEN_SRC): $(TOOL_GEN_HEADER)
 $(TOOL_TEST_OBJS): $(TOOL_GEN_SRC)
-$(TOOL_TEST_PROGRAMS): $(TOOL_TEST_COMMON_OBJS) tools/test/gen_test.o
+$(TOOL_TEST_PROGRAMS): $(TOOL_TEST_COMMON_OBJS) $(TOOL_GEN_SRC:.c=.o)
 $(TOOL_GEN_SRC) $(TOOL_GEN_HEADER): $(TOOLS_WIRE_DEPS)
 
 tools/test/gen_test.h:
@@ -30,11 +30,23 @@ tools/test/gen_test.c:
 	tools/generate-bolts.py --page impl ${@:.c=.h} test_type < tools/test/test_cases > $@
 	@tools/update-mocks.sh "$@"
 
+tools/test/gen_print.h: wire/gen_onion_wire.h
+	tools/generate-bolts.py -P --page header $@ test_type < tools/test/test_cases > $@
+
+tools/test/gen_print.c:
+	echo '#include "gen_test.h"' > $@
+	tools/generate-bolts.py -P --page impl ${@:.c=.h} test_type < tools/test/test_cases >> $@
+
+
 ALL_TEST_PROGRAMS += $(TOOL_TEST_PROGRAMS)
 
 check-tools: $(TOOL_TEST_PROGRAMS:%=unittest/%)
 
+update-mocks: tools-update-mocks
+
+tools-update-mocks: $(TOOL_TEST_SRC:%=update-mocks/%)
+
 clean: tools-test-clean
 
 tools-test-clean:
-	$(RM) $(TOOL_GEN_FILES) $(TOOL_TEST_OBJS)
+	$(RM) $(TOOL_GEN_HEADER) $(TOOL_GEN_SRC) $(TOOL_TEST_OBJS)

tools/test/run-test-wire.c

@@ -1,4 +1,5 @@
 #include "gen_test.h"
+#include "gen_print.h"
 
 #include <assert.h>
 #include <stdio.h>
@@ -6,6 +7,28 @@
 #include <common/utils.h>
 
 /* AUTOGENERATED MOCKS START */
+/* Generated stub for fromwire_peektype */
+int fromwire_peektype(const u8 *cursor UNNEEDED)
+{ fprintf(stderr, "fromwire_peektype called!\n"); abort(); }
+/* Generated stub for printwire_amount_msat */
+void printwire_amount_msat(const char *fieldname UNNEEDED, const struct amount_msat *msat UNNEEDED)
+{ fprintf(stderr, "printwire_amount_msat called!\n"); abort(); }
+/* Generated stub for printwire_tlvs */
+void printwire_tlvs(const char *tlv_name UNNEEDED, const u8 **cursor UNNEEDED, size_t *plen UNNEEDED,
+		    const struct tlv_print_record_type types[] UNNEEDED, size_t num_types UNNEEDED)
+{ fprintf(stderr, "printwire_tlvs called!\n"); abort(); }
+/* Generated stub for printwire_u16 */
+void printwire_u16(const char *fieldname UNNEEDED, const u16 *v UNNEEDED)
+{ fprintf(stderr, "printwire_u16 called!\n"); abort(); }
+/* Generated stub for printwire_u32 */
+void printwire_u32(const char *fieldname UNNEEDED, const u32 *v UNNEEDED)
+{ fprintf(stderr, "printwire_u32 called!\n"); abort(); }
+/* Generated stub for printwire_u64 */
+void printwire_u64(const char *fieldname UNNEEDED, const u64 *v UNNEEDED)
+{ fprintf(stderr, "printwire_u64 called!\n"); abort(); }
+/* Generated stub for printwire_u8_array */
+void printwire_u8_array(const char *fieldname UNNEEDED, const u8 **cursor UNNEEDED, size_t *plen UNNEEDED, size_t len UNNEEDED)
+{ fprintf(stderr, "printwire_u8_array called!\n"); abort(); }
 /* AUTOGENERATED MOCKS END */
 
 int main(void)

wire/tlvstream.h

@@ -11,7 +11,7 @@ struct tlv_record_type {
 	/* If this type is present return marshalled value.  Otherwise
 	 * returns NULL. */
 	u8 *(*towire)(const tal_t *ctx, const void *record);
-	/* Must call fromwire_fail() it can't parse. */
+	/* Must call fromwire_fail() if it can't parse. */
 	void (*fromwire)(const u8 **cursor, size_t *max, void *record);
 };