Merge pull request #132 from ElementsProject/pr-130-refinements

PR #130 refinements

lightningd/Makefile

@@ -45,6 +45,7 @@ LIGHTNINGD_LIB_SRC :=				\
 	lightningd/debug.c			\
 	lightningd/derive_basepoints.c		\
 	lightningd/funding_tx.c			\
+	lightningd/gossip_msg.c			\
 	lightningd/htlc_tx.c			\
 	lightningd/key_derive.c			\
 	lightningd/msg_queue.c			\

lightningd/channel/channel_wire.csv

@@ -19,25 +19,25 @@ channel_normal_operation,1001
 # Begin!  You're still waiting for the tx to be buried though (passes
 # gossipd-client fd)
 channel_init,1
-channel_init,0,funding_txid,32,struct sha256_double
+channel_init,0,funding_txid,struct sha256_double
 channel_init,32,funding_txout,2
-channel_init,34,our_config,36,struct channel_config
+channel_init,34,our_config,struct channel_config
-channel_init,70,their_config,36,struct channel_config
+channel_init,70,their_config,struct channel_config
-channel_init,106,first_commit_sig,64,secp256k1_ecdsa_signature
+channel_init,106,first_commit_sig,secp256k1_ecdsa_signature
-channel_init,166,crypto_state,144,struct crypto_state
+channel_init,166,crypto_state,struct crypto_state
 channel_init,310,remote_fundingkey,33
 channel_init,343,revocation_basepoint,33
 channel_init,376,payment_basepoint,33
 channel_init,409,delayed_payment_basepoint,33
 channel_init,442,their_per_commit_point,33
-channel_init,475,am_funder,1,bool
+channel_init,475,am_funder,bool
 channel_init,476,feerate,4
 channel_init,480,funding_satoshi,8
 channel_init,488,push_msat,8
-channel_init,496,seed,32,struct privkey
+channel_init,496,seed,struct privkey
-channel_init,529,local_node_id,33,struct pubkey
+channel_init,529,local_node_id,struct pubkey
-channel_init,562,remote_node_id,33,struct pubkey
+channel_init,562,remote_node_id,struct pubkey
 
 # Tx is deep enough, go!
 channel_funding_locked,2
-channel_funding_locked,0,short_channel_id,8,struct short_channel_id
+channel_funding_locked,0,short_channel_id,struct short_channel_id

lightningd/gossip/gossip.c

@@ -20,6 +20,7 @@
 #include <lightningd/cryptomsg.h>
 #include <lightningd/debug.h>
 #include <lightningd/gossip/gen_gossip_wire.h>
+#include <lightningd/gossip_msg.h>
 #include <secp256k1_ecdh.h>
 #include <sodium/randombytes.h>
 #include <status.h>
@@ -438,6 +439,30 @@ static struct io_plan *release_peer(struct io_conn *conn, struct daemon *daemon,
 		      "Unknown peer %"PRIu64, unique_id);
 }
 
+static struct io_plan *getnodes(struct io_conn *conn, struct daemon *daemon)
+{
+	tal_t *tmpctx = tal_tmpctx(daemon);
+	u8 *out;
+	struct node *n;
+	struct node_map_iter i;
+	struct gossip_getnodes_entry *nodes;
+	size_t node_count = 0;
+
+	nodes = tal_arr(tmpctx, struct gossip_getnodes_entry, node_count);
+	n = node_map_first(daemon->rstate->nodes, &i);
+	while (n != NULL) {
+		tal_resize(&nodes, node_count + 1);
+		nodes[node_count].nodeid = n->id;
+		nodes[node_count].hostname = n->hostname;
+		nodes[node_count].port = n->port;
+		node_count++;
+		n = node_map_next(daemon->rstate->nodes, &i);
+	}
+	out = towire_gossip_getnodes_reply(daemon, nodes);
+	tal_free(tmpctx);
+	return io_write_wire(conn, take(out), next_req_in, daemon);
+}
+
 static struct io_plan *recv_req(struct io_conn *conn, struct daemon *daemon)
 {
 	enum gossip_wire_type t = fromwire_peektype(daemon->msg_in);
@@ -451,7 +476,11 @@ static struct io_plan *recv_req(struct io_conn *conn, struct daemon *daemon)
 	case WIRE_GOSSIPCTL_RELEASE_PEER:
 		return release_peer(conn, daemon, daemon->msg_in);
 
+	case WIRE_GOSSIP_GETNODES_REQUEST:
+		return getnodes(conn, daemon);
+
 	case WIRE_GOSSIPCTL_RELEASE_PEER_REPLY:
+	case WIRE_GOSSIP_GETNODES_REPLY:
 	case WIRE_GOSSIPSTATUS_INIT_FAILED:
 	case WIRE_GOSSIPSTATUS_BAD_NEW_PEER_REQUEST:
 	case WIRE_GOSSIPSTATUS_BAD_RELEASE_REQUEST:

lightningd/gossip/gossip_wire.csv

@@ -9,7 +9,7 @@ gossipstatus_fdpass_failed,0x8004
 gossipstatus_peer_bad_msg,1000
 gossipstatus_peer_bad_msg,0,unique_id,8
 gossipstatus_peer_bad_msg,8,len,2
-gossipstatus_peer_bad_msg,10,err,len,u8
+gossipstatus_peer_bad_msg,10,err,len*u8
 
 #include <lightningd/cryptomsg.h>
 
@@ -17,7 +17,7 @@ gossipstatus_peer_bad_msg,10,err,len,u8
 # (if it is to move onto a channel, we get a status msg).
 gossipctl_new_peer,1
 gossipctl_new_peer,0,unique_id,8
-gossipctl_new_peer,8,crypto_state,144,struct crypto_state
+gossipctl_new_peer,8,crypto_state,struct crypto_state
 
 # Tell it to release a peer which has initialized.
 gossipctl_release_peer,2
@@ -26,7 +26,7 @@ gossipctl_release_peer,0,unique_id,8
 # This releases the peer and returns the cryptostate (followed by fd)
 gossipctl_release_peer_reply,102
 gossipctl_release_peer_reply,0,unique_id,8
-gossipctl_release_peer_reply,8,crypto_state,144,struct crypto_state
+gossipctl_release_peer_reply,8,crypto_state,struct crypto_state
 
 # This is where we save a peer's features.
 #gossipstatus_peer_features,1
@@ -43,6 +43,14 @@ gossipstatus_peer_ready,0,unique_id,8
 # Peer can send non-gossip packet (usually an open_channel) (followed by fd)
 gossipstatus_peer_nongossip,4
 gossipstatus_peer_nongossip,0,unique_id,8
-gossipstatus_peer_nongossip,10,crypto_state,144,struct crypto_state
+gossipstatus_peer_nongossip,10,crypto_state,struct crypto_state
 gossipstatus_peer_nongossip,154,len,2
-gossipstatus_peer_nongossip,156,msg,len,u8
+gossipstatus_peer_nongossip,156,msg,len*u8
+
+# Pass JSON-RPC getnodes call through
+gossip_getnodes_request,5
+
+#include <lightningd/gossip_msg.h>
+gossip_getnodes_reply,105
+gossip_getnodes_reply,0,num_nodes,u16
+gossip_getnodes_reply,2,nodes,num_nodes*struct gossip_getnodes_entry

lightningd/gossip_control.c

@@ -9,6 +9,7 @@
 #include <inttypes.h>
 #include <lightningd/cryptomsg.h>
 #include <lightningd/gossip/gen_gossip_wire.h>
+#include <lightningd/gossip_msg.h>
 #include <wire/gen_peer_wire.h>
 
 static void gossip_finished(struct subd *gossip, int status)
@@ -118,8 +119,10 @@ static enum subd_msg_ret gossip_msg(struct subd *gossip,
 	/* These are messages we send, not them. */
 	case WIRE_GOSSIPCTL_NEW_PEER:
 	case WIRE_GOSSIPCTL_RELEASE_PEER:
+	case WIRE_GOSSIP_GETNODES_REQUEST:
 	/* This is a reply, so never gets through to here. */
 	case WIRE_GOSSIPCTL_RELEASE_PEER_REPLY:
+	case WIRE_GOSSIP_GETNODES_REPLY:
 		break;
 	case WIRE_GOSSIPSTATUS_PEER_BAD_MSG:
 		peer_bad_message(gossip, msg);
@@ -147,3 +150,48 @@ void gossip_init(struct lightningd *ld)
 	if (!ld->gossip)
 		err(1, "Could not subdaemon gossip");
 }
+
+static bool json_getnodes_reply(struct subd *gossip, const u8 *reply,
+				struct command *cmd)
+{
+	struct gossip_getnodes_entry *nodes;
+	struct json_result *response = new_json_result(cmd);
+	size_t i;
+
+	if (!fromwire_gossip_getnodes_reply(reply, reply, NULL, &nodes)) {
+		command_fail(cmd, "Malformed gossip_getnodes response");
+		return true;
+	}
+
+	json_object_start(response, NULL);
+	json_array_start(response, "nodes");
+
+	for (i = 0; i < tal_count(nodes); i++) {
+		json_object_start(response, NULL);
+		json_add_pubkey(response, "nodeid", &nodes[i].nodeid);
+		if (tal_len(nodes[i].hostname) > 0) {
+			json_add_string(response, "hostname", nodes[i].hostname);
+		} else {
+			json_add_null(response, "hostname");
+		}
+		json_add_num(response, "port", nodes[i].port);
+		json_object_end(response);
+	}
+	json_array_end(response);
+	json_object_end(response);
+	command_success(cmd, response);
+	return true;
+}
+
+static void json_getnodes(struct command *cmd, const char *buffer,
+			  const jsmntok_t *params)
+{
+	struct lightningd *ld = ld_from_dstate(cmd->dstate);
+	u8 *req = towire_gossip_getnodes_request(cmd);
+	subd_req(ld->gossip, req, -1, NULL, json_getnodes_reply, cmd);
+}
+
+static const struct json_command getnodes_command = {
+    "getnodes", json_getnodes, "Retrieve all nodes in our local network view",
+    "Returns a list of all nodes that we know about"};
+AUTODATA(json_command, &getnodes_command);

lightningd/gossip_msg.c

@@ -0,0 +1,27 @@
+#include <lightningd/gossip_msg.h>
+#include <wire/wire.h>
+
+void fromwire_gossip_getnodes_entry(const u8 **pptr, size_t *max, struct gossip_getnodes_entry *entry)
+{
+	u8 hostnamelen;
+	fromwire_pubkey(pptr, max, &entry->nodeid);
+	hostnamelen = fromwire_u8(pptr, max);
+	entry->hostname = tal_arr(entry, char, hostnamelen);
+	fromwire_u8_array(pptr, max, (u8*)entry->hostname, hostnamelen);
+	entry->port = fromwire_u16(pptr, max);
+}
+void towire_gossip_getnodes_entry(u8 **pptr, const struct gossip_getnodes_entry *entry)
+{
+	u8 hostnamelen;
+	towire_pubkey(pptr, &entry->nodeid);
+	if (entry->hostname) {
+		hostnamelen = strlen(entry->hostname);
+		towire_u8(pptr, hostnamelen);
+		towire_u8_array(pptr, (u8*)entry->hostname, hostnamelen);
+	}else {
+		/* If we don't have a hostname just write an empty string */
+		hostnamelen = 0;
+		towire_u8(pptr, hostnamelen);
+	}
+	towire_u16(pptr, entry->port);
+}

lightningd/gossip_msg.h

@@ -0,0 +1,15 @@
+#ifndef LIGHTNING_LIGHTNINGD_GOSSIP_MSG_H
+#define LIGHTNING_LIGHTNINGD_GOSSIP_MSG_H
+#include "config.h"
+#include <bitcoin/pubkey.h>
+
+struct gossip_getnodes_entry {
+	struct pubkey nodeid;
+	char *hostname;
+	u16 port;
+};
+
+void fromwire_gossip_getnodes_entry(const u8 **pptr, size_t *max, struct gossip_getnodes_entry *entry);
+void towire_gossip_getnodes_entry(u8 **pptr, const struct gossip_getnodes_entry *entry);
+
+#endif /* LIGHTNING_LIGHTGNINGD_GOSSIP_MSG_H */

lightningd/handshake/handshake_wire.csv

@@ -36,12 +36,12 @@ handshake_responder,1
 handshake_responder,1,my_id,33
 handshake_responder_reply,101
 handshake_responder_reply,0,initiator_id,33
-handshake_responder_reply,33,cs,144,struct crypto_state
+handshake_responder_reply,33,cs,struct crypto_state
 
 handshake_initiator,2
 handshake_initiator,0,my_id,33
 handshake_initiator,33,responder_id,33
 
 handshake_initiator_reply,102
-handshake_initiator_reply,0,cs,144,struct crypto_state
+handshake_initiator_reply,0,cs,struct crypto_state
 

lightningd/hsm/hsm_wire.csv

@@ -9,17 +9,17 @@ hsmstatus_key_failed,0x8004
 hsmstatus_client_bad_request,1000
 hsmstatus_client_bad_request,0,unique-id,8
 hsmstatus_client_bad_request,8,len,2
-hsmstatus_client_bad_request,10,msg,len,u8
+hsmstatus_client_bad_request,10,msg,len*u8
 
 # Start the HSM.
 hsmctl_init,1
-hsmctl_init,0,new,1,bool
+hsmctl_init,0,new,bool
 
 hsmctl_init_reply,101
 hsmctl_init_reply,0,node_id,33
-hsmctl_init_reply,33,peer_seed,32,struct privkey
+hsmctl_init_reply,33,peer_seed,struct privkey
 hsmctl_init_reply,65,bip32_len,2
-hsmctl_init_reply,67,bip32_seed,bip32_len*1,u8
+hsmctl_init_reply,67,bip32_seed,bip32_len*u8
 
 # ECDH returns an fd.
 hsmctl_hsmfd_ecdh,3
@@ -38,10 +38,10 @@ hsmctl_sign_funding,16,change_keyindex,4
 hsmctl_sign_funding,20,our_pubkey,33
 hsmctl_sign_funding,52,their_pubkey,33
 hsmctl_sign_funding,85,num_inputs,2
-hsmctl_sign_funding,87,inputs,num_inputs*49,struct utxo
+hsmctl_sign_funding,87,inputs,num_inputs*struct utxo
 
 hsmctl_sign_funding_reply,104
 hsmctl_sign_funding_reply,0,num_sigs,2
-hsmctl_sign_funding_reply,0,sig,num_sigs*64,secp256k1_ecdsa_signature
+hsmctl_sign_funding_reply,0,sig,num_sigs*secp256k1_ecdsa_signature
 
 

lightningd/opening/opening_wire.csv

@@ -15,13 +15,13 @@ opening_peer_bad_initial_message,0x8014
 #include <lightningd/channel_config.h>
 opening_init,0
 # Base configuration we'll offer (channel reserve will vary with amount)
-opening_init,0,our_config,36,struct channel_config
+opening_init,0,our_config,struct channel_config
 # Minimum/maximum configuration values we'll accept
 opening_init,36,max_to_self_delay,4
 opening_init,40,min_effective_htlc_capacity_msat,8
-opening_init,48,crypto_state,144,struct crypto_state
+opening_init,48,crypto_state,struct crypto_state
 # Seed to generate all the keys from
-opening_init,196,seed,32,struct privkey
+opening_init,196,seed,struct privkey
 
 # This means we offer the open.
 opening_open,1
@@ -37,14 +37,14 @@ opening_open_reply,0,remote_fundingkey,33
 
 # Now we give the funding txid and outnum.
 opening_open_funding,2
-opening_open_funding,0,txid,32,struct sha256_double
+opening_open_funding,0,txid,struct sha256_double
-opening_open_funding,32,txout,2,u16
+opening_open_funding,32,txout,u16
 
 # This gives their sig, means we can broadcast tx: we're done.
 opening_open_funding_reply,102
-opening_open_funding_reply,0,their_config,36,struct channel_config
+opening_open_funding_reply,0,their_config,struct channel_config
-opening_open_funding_reply,36,first_commit_sig,64,secp256k1_ecdsa_signature
+opening_open_funding_reply,36,first_commit_sig,secp256k1_ecdsa_signature
-opening_open_funding_reply,100,crypto_state,144,struct crypto_state
+opening_open_funding_reply,100,crypto_state,struct crypto_state
 opening_open_funding_reply,244,revocation_basepoint,33
 opening_open_funding_reply,277,payment_basepoint,33
 opening_open_funding_reply,310,delayed_payment_basepoint,33
@@ -55,20 +55,20 @@ opening_accept,3
 opening_accept,0,min_feerate,4
 opening_accept,4,max_feerate,4
 opening_accept,8,len,2
-opening_accept,10,msg,len,u8
+opening_accept,10,msg,len*u8
 
 # This gives the txid of their funding tx to watch.
 opening_accept_reply,103
-opening_accept_reply,0,funding_txid,32,struct sha256_double
+opening_accept_reply,0,funding_txid,struct sha256_double
 
 # Acknowledge watch is in place, now can send sig.
 opening_accept_finish,4
 
 opening_accept_finish_reply,104
-opening_accept_finish_reply,32,funding_txout,2,u16
+opening_accept_finish_reply,32,funding_txout,u16
-opening_accept_finish_reply,0,their_config,36,struct channel_config
+opening_accept_finish_reply,0,their_config,struct channel_config
-opening_accept_finish_reply,36,first_commit_sig,64,secp256k1_ecdsa_signature
+opening_accept_finish_reply,36,first_commit_sig,secp256k1_ecdsa_signature
-opening_accept_finish_reply,100,crypto_state,144,struct crypto_state
+opening_accept_finish_reply,100,crypto_state,struct crypto_state
 opening_accept_finish_reply,244,remote_fundingkey,33
 opening_accept_finish_reply,277,revocation_basepoint,33
 opening_accept_finish_reply,310,payment_basepoint,33

lightningd/utxo.c

@@ -18,20 +18,3 @@ void fromwire_utxo(const u8 **ptr, size_t *max, struct utxo *utxo)
 	utxo->keyindex = fromwire_u32(ptr, max);
 	utxo->is_p2sh = fromwire_bool(ptr, max);
 }
-
-void fromwire_utxo_array(const u8 **ptr, size_t *max,
-			 struct utxo *utxo, size_t num)
-{
-	size_t i;
-
-	for (i = 0; i < num; i++)
-		fromwire_utxo(ptr, max, &utxo[i]);
-}
-
-void towire_utxo_array(u8 **pptr, const struct utxo *utxo, size_t num)
-{
-	size_t i;
-
-	for (i = 0; i < num; i++)
-		towire_utxo(pptr, &utxo[i]);
-}

lightningd/utxo.h

@@ -16,9 +16,4 @@ struct utxo {
 
 void towire_utxo(u8 **pptr, const struct utxo *utxo);
 void fromwire_utxo(const u8 **ptr, size_t *max, struct utxo *utxo);
-
-void fromwire_utxo_array(const u8 **ptr, size_t *max,
-			 struct utxo *utxo, size_t num);
-
-void towire_utxo_array(u8 **pptr, const struct utxo *utxo, size_t num);
 #endif /* LIGHTNING_LIGHTNINGD_UTXO_H */

tools/generate-wire.py

@@ -31,6 +31,10 @@ class FieldType(object):
     def is_assignable(self):
         return self.name in ['u8', 'u16', 'u32', 'u64', 'bool']
 
+    # We only accelerate the u8 case: it's common and trivial.
+    def has_array_helper(self):
+        return self.name in ['u8']
+
     # Returns base size
     @staticmethod
     def _typesize(typename):
@@ -77,40 +81,48 @@ sizetypemap = {
     1: FieldType('u8')
 }
 
+# It would be nicer if we had put '*1' in spec and disallowed bare lenvar.
+# In practice we only recognize raw lenvar when it's the previous field.
+
+# size := baresize | arraysize
+# baresize := simplesize | lenvar
+# simplesize := number | type
+# arraysize := lenvar '*' simplesize
 class Field(object):
-    def __init__(self, message, name, size, comments, typename=None):
+    def __init__(self, message, name, size, comments, prevname):
         self.message = message
         self.comments = comments
         self.name = name.replace('-', '_')
         self.is_len_var = False
         self.lenvar = None
+        self.num_elems = 1
 
-        # Size could be a literal number (eg. 33), or a field (eg 'len'), or
+        # If it's an arraysize, swallow prefix.
-        # a multiplier of a field (eg. num-htlc-timeouts*64).
+        if '*' in size:
-        try:
+            self.lenvar = size.split('*')[0].replace('-','_')
-            base_size = int(size)
+            size = size.split('*')[1]
-        except ValueError:
-            # If it's a multiplicitive expression, must end in basesize.
-            if '*' in size:
-                base_size = int(size.split('*')[1])
-                self.lenvar = size.split('*')[0]
-            else:
-                base_size = 0
-                self.lenvar = size
-            self.lenvar = self.lenvar.replace('-','_')
-
-        if typename is None:
-            self.fieldtype = Field._guess_type(message,self.name,base_size)
         else:
-            self.fieldtype = FieldType(typename)
+            if size == prevname:
+                # Raw length field, implies u8.
+                self.lenvar = size.replace('-','_')
+                size = 'u8'
 
-        # Unknown types are assumed to have base_size: div by 0 if that's unknown.
+        try:
-        if self.fieldtype.tsize == 0:
+            # Just a number?  Guess based on size.
-            self.fieldtype.tsize = base_size
+            base_size = int(size)
+            self.fieldtype = Field._guess_type(message,self.name,base_size)
+            # There are some arrays which we have to guess, based on sizes.
+            if base_size % self.fieldtype.tsize != 0:
+                raise ValueError('Invalid size {} for {}.{} not a multiple of {}'
+                                 .format(base_size,
+                                         self.message,
+                                         self.name,
+                                         self.fieldtype.tsize))
+            self.num_elems = int(base_size / self.fieldtype.tsize)
 
-        if base_size % self.fieldtype.tsize != 0:
+        except ValueError:
-            raise ValueError('Invalid size {} for {}.{} not a multiple of {}'.format(base_size,self.message,self.name,self.fieldtype.tsize))
+            # Not a number; must be a type.
-        self.num_elems = int(base_size / self.fieldtype.tsize)
+            self.fieldtype = FieldType(size)
 
     def is_padding(self):
         return self.name.startswith('pad')
@@ -127,6 +139,9 @@ class Field(object):
             return False
         return self.fieldtype.is_assignable()
 
+    def has_array_helper(self):
+        return self.fieldtype.has_array_helper()
+    
     # Returns FieldType
     @staticmethod
     def _guess_type(message, fieldname, base_size):
@@ -209,6 +224,20 @@ class Message(object):
             self.has_variable_fields = True
         self.fields.append(field)
 
+    def print_fromwire_array(self, subcalls, basetype, f, name, num_elems):
+        if f.has_array_helper():
+            subcalls.append('\tfromwire_{}_array(&cursor, plen, {}, {});'
+                            .format(basetype, name, num_elems))
+        else:
+            subcalls.append('\tfor (size_t i = 0; i < {}; i++)'
+                            .format(num_elems))
+            if f.is_assignable():
+                subcalls.append('\t\t{}[i] = fromwire_{}(&cursor, plen);'
+                                .format(name, basetype))
+            else:
+                subcalls.append('\t\tfromwire_{}(&cursor, plen, {} + i);'
+                                .format(basetype, name))
+
     def print_fromwire(self,is_header):
         ctx_arg = 'const tal_t *ctx, ' if self.has_variable_fields else ''
 
@@ -240,15 +269,15 @@ class Message(object):
                 subcalls.append('\tfromwire_pad(&cursor, plen, {});'
                                 .format(f.num_elems))
             elif f.is_array():
-                subcalls.append("\t//1th case {name}".format(name=f.name))
+                self.print_fromwire_array(subcalls, basetype, f, f.name,
-                subcalls.append('\tfromwire_{}_array(&cursor, plen, {}, {});'
+                                          f.num_elems)
-                                .format(basetype, f.name, f.num_elems))
             elif f.is_variable_size():
                 subcalls.append("\t//2th case {name}".format(name=f.name))
                 subcalls.append('\t*{} = tal_arr(ctx, {}, {});'
                                 .format(f.name, f.fieldtype.name, f.lenvar))
-                subcalls.append('\tfromwire_{}_array(&cursor, plen, *{}, {});'
+
-                                .format(basetype, f.name, f.lenvar))
+                self.print_fromwire_array(subcalls, basetype, f, '*'+f.name,
+                                          f.lenvar)
             elif f.is_assignable():
                 subcalls.append("\t//3th case {name}".format(name=f.name))
                 if f.is_len_var:
@@ -271,6 +300,15 @@ class Message(object):
             subcalls='\n'.join(subcalls)
         )
 
+    def print_towire_array(self, subcalls, basetype, f, num_elems):
+        if f.has_array_helper():
+            subcalls.append('\ttowire_{}_array(&p, {}, {});'
+                            .format(basetype, f.name, num_elems))
+        else:
+            subcalls.append('\tfor (size_t i = 0; i < {}; i++)\n'
+                            '\t\ttowire_{}(&p, {} + i);'
+                            .format(num_elems, basetype, f.name))
+
     def print_towire(self,is_header):
         template = towire_header_templ if is_header else towire_impl_templ
         args = []
@@ -304,11 +342,9 @@ class Message(object):
                 subcalls.append('\ttowire_pad(&p, {});'
                       .format(f.num_elems))
             elif f.is_array():
-                subcalls.append('\ttowire_{}_array(&p, {}, {});'
+                self.print_towire_array(subcalls, basetype, f, f.num_elems)
-                      .format(basetype, f.name, f.num_elems))
             elif f.is_variable_size():
-                subcalls.append('\ttowire_{}_array(&p, {}, {});'
+                self.print_towire_array(subcalls, basetype, f, f.lenvar)
-                      .format(basetype, f.name, f.lenvar))
             else:
                 subcalls.append('\ttowire_{}(&p, {});'
                       .format(basetype, f.name))
@@ -332,6 +368,7 @@ options = parser.parse_args()
 messages = []
 comments = []
 includes = []
+prevfield = None
 
 # Read csv lines.  Single comma is the message values, more is offset/len.
 for line in fileinput.input(options.files):
@@ -354,18 +391,19 @@ for line in fileinput.input(options.files):
         # eg commit_sig,132
         messages.append(Message(parts[0],Enumtype("WIRE_" + parts[0].upper(), parts[1]), comments))
         comments=[]
-    else:
+        prevfield = None
+    elif len(parts) == 4:
         # eg commit_sig,0,channel-id,8 OR
-        #    commit_sig,0,channel-id,8,u64
+        #    commit_sig,0,channel-id,u64
         for m in messages:
             if m.name == parts[0]:
-                if len(parts) == 4:
+                m.addField(Field(parts[0], parts[2], parts[3], comments, prevfield))
-                    m.addField(Field(parts[0], parts[2], parts[3], comments))
+                prevfield = parts[2]
-                else:
-                    m.addField(Field(parts[0], parts[2], parts[3], comments,
-                                     parts[4]))
                 break
         comments=[]
+    else:
+        raise ValueError('Line {} malformed'.format(line.rstrip()))
+        
 
 header_template = """#ifndef LIGHTNING_{idem}
 #define LIGHTNING_{idem}

wire/fromwire.c

@@ -160,53 +160,11 @@ void fromwire_u8_array(const u8 **cursor, size_t *max, u8 *arr, size_t num)
 	fromwire(cursor, max, arr, num);
 }
 
-void fromwire_u32_array(const u8 **cursor, size_t *max, u32 *arr, size_t num)
-{
-	size_t i;
-
-	for (i = 0; i < num; i++)
-		arr[i] = fromwire_u32(cursor, max);
-}
-
-void fromwire_u64_array(const u8 **cursor, size_t *max, u64 *arr, size_t num)
-{
-	size_t i;
-
-	for (i = 0; i < num; i++)
-		arr[i] = fromwire_u64(cursor, max);
-}
-
-void fromwire_bool_array(const u8 **cursor, size_t *max, bool *arr, size_t num)
-{
-	size_t i;
-
-	for (i = 0; i < num; i++)
-		arr[i] = fromwire_bool(cursor, max);
-}
-
 void fromwire_pad(const u8 **cursor, size_t *max, size_t num)
 {
 	fromwire(cursor, max, NULL, num);
 }
 
-void fromwire_secp256k1_ecdsa_signature_array(const u8 **cursor, size_t *max,
-			      secp256k1_ecdsa_signature *arr, size_t num)
-{
-	size_t i;
-
-	for (i = 0; i < num; i++)
-		fromwire_secp256k1_ecdsa_signature(cursor, max, arr + i);
-}
-
-void fromwire_sha256_double_array(const u8 **cursor, size_t *max,
-				  struct sha256_double *arr, size_t num)
-{
-	size_t i;
-
-	for (i = 0; i < num; i++)
-		fromwire_sha256_double(cursor, max, arr + i);
-}
-
 static char *fmt_short_channel_id(const tal_t *ctx,
 				  const struct short_channel_id *id)
 {

wire/towire.c

@@ -104,30 +104,6 @@ void towire_u8_array(u8 **pptr, const u8 *arr, size_t num)
 	towire(pptr, arr, num);
 }
 
-void towire_u32_array(u8 **pptr, const u32 *arr, size_t num)
-{
-	size_t i;
-
-	for (i = 0; i < num; i++)
-		towire_u32(pptr, arr[i]);
-}
-
-void towire_u64_array(u8 **pptr, const u64 *arr, size_t num)
-{
-	size_t i;
-
-	for (i = 0; i < num; i++)
-		towire_u64(pptr, arr[i]);
-}
-
-void towire_bool_array(u8 **pptr, const bool *arr, size_t num)
-{
-	size_t i;
-
-	for (i = 0; i < num; i++)
-		towire_bool(pptr, arr[i]);
-}
-
 void towire_pad(u8 **pptr, size_t num)
 {
 	/* Simply insert zeros. */
@@ -136,21 +112,3 @@ void towire_pad(u8 **pptr, size_t num)
 	tal_resize(pptr, oldsize + num);
 	memset(*pptr + oldsize, 0, num);
 }
-
-void towire_secp256k1_ecdsa_signature_array(u8 **pptr,
-			    const secp256k1_ecdsa_signature *arr, size_t num)
-{
-	size_t i;
-
-	for (i = 0; i < num; i++)
-		towire_secp256k1_ecdsa_signature(pptr, arr+i);
-}
-
-void towire_sha256_double_array(u8 **pptr,
-				const struct sha256_double *arr, size_t num)
-{
-	size_t i;
-
-	for (i = 0; i < num; i++)
-		towire_sha256_double(pptr, arr+i);
-}

wire/wire.h

@@ -43,14 +43,6 @@ void towire_pad(u8 **pptr, size_t num);
 void towire_bool(u8 **pptr, bool v);
 
 void towire_u8_array(u8 **pptr, const u8 *arr, size_t num);
-void towire_u32_array(u8 **pptr, const u32 *arr, size_t num);
-void towire_u64_array(u8 **pptr, const u64 *arr, size_t num);
-void towire_bool_array(u8 **pptr, const bool *arr, size_t num);
-void towire_secp256k1_ecdsa_signature_array(u8 **pptr,
-			const secp256k1_ecdsa_signature *arr, size_t num);
-void towire_sha256_double_array(u8 **pptr,
-				const struct sha256_double *arr, size_t num);
-
 
 const u8 *fromwire(const u8 **cursor, size_t *max, void *copy, size_t n);
 u8 fromwire_u8(const u8 **cursor, size_t *max);
@@ -73,12 +65,4 @@ void fromwire_ipv6(const u8 **cursor, size_t *max, struct ipv6 *ipv6);
 void fromwire_pad(const u8 **cursor, size_t *max, size_t num);
 
 void fromwire_u8_array(const u8 **cursor, size_t *max, u8 *arr, size_t num);
-void fromwire_u32_array(const u8 **cursor, size_t *max, u32 *arr, size_t num);
-void fromwire_u64_array(const u8 **cursor, size_t *max, u64 *arr, size_t num);
-void fromwire_bool_array(const u8 **cursor, size_t *max, bool *arr, size_t num);
-
-void fromwire_secp256k1_ecdsa_signature_array(const u8 **cursor, size_t *max,
-			      secp256k1_ecdsa_signature *arr, size_t num);
-void fromwire_sha256_double_array(const u8 **cursor, size_t *max,
-				  struct sha256_double *arr, size_t num);
 #endif /* LIGHTNING_WIRE_WIRE_H */