From e042198cf8c94768c199eb2af07d2f8b160184b2 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 16 Mar 2017 14:35:25 +1030 Subject: [PATCH] tools/generate-wire.py: allow typename instead of type sizes. We use the fourth value (size) to determine the type, unless the fifth value is suppled. That's silly: allow the fourth value to be a typename, since that's the only reason we care about the size at all! Unfortunately there are places in the spec where we use a raw fieldname without '*1' for a length, so we have to distingish this from the typename case. Signed-off-by: Rusty Russell --- lightningd/channel/channel_wire.csv | 20 +++---- lightningd/gossip/gossip_wire.csv | 14 ++--- lightningd/handshake/handshake_wire.csv | 4 +- lightningd/hsm/hsm_wire.csv | 12 ++--- lightningd/opening/opening_wire.csv | 28 +++++----- tools/generate-wire.py | 72 ++++++++++++++----------- 6 files changed, 80 insertions(+), 70 deletions(-) diff --git a/lightningd/channel/channel_wire.csv b/lightningd/channel/channel_wire.csv index bb07a0b2f..e895feddb 100644 --- a/lightningd/channel/channel_wire.csv +++ b/lightningd/channel/channel_wire.csv @@ -19,25 +19,25 @@ channel_normal_operation,1001 # Begin! You're still waiting for the tx to be buried though (passes # gossipd-client fd) channel_init,1 -channel_init,0,funding_txid,32,struct sha256_double +channel_init,0,funding_txid,struct sha256_double channel_init,32,funding_txout,2 -channel_init,34,our_config,36,struct channel_config -channel_init,70,their_config,36,struct channel_config -channel_init,106,first_commit_sig,64,secp256k1_ecdsa_signature -channel_init,166,crypto_state,144,struct crypto_state +channel_init,34,our_config,struct channel_config +channel_init,70,their_config,struct channel_config +channel_init,106,first_commit_sig,secp256k1_ecdsa_signature +channel_init,166,crypto_state,struct crypto_state channel_init,310,remote_fundingkey,33 channel_init,343,revocation_basepoint,33 channel_init,376,payment_basepoint,33 channel_init,409,delayed_payment_basepoint,33 channel_init,442,their_per_commit_point,33 -channel_init,475,am_funder,1,bool +channel_init,475,am_funder,bool channel_init,476,feerate,4 channel_init,480,funding_satoshi,8 channel_init,488,push_msat,8 -channel_init,496,seed,32,struct privkey -channel_init,529,local_node_id,33,struct pubkey -channel_init,562,remote_node_id,33,struct pubkey +channel_init,496,seed,struct privkey +channel_init,529,local_node_id,struct pubkey +channel_init,562,remote_node_id,struct pubkey # Tx is deep enough, go! channel_funding_locked,2 -channel_funding_locked,0,short_channel_id,8,struct short_channel_id \ No newline at end of file +channel_funding_locked,0,short_channel_id,struct short_channel_id diff --git a/lightningd/gossip/gossip_wire.csv b/lightningd/gossip/gossip_wire.csv index d6f3a3135..b162816e5 100644 --- a/lightningd/gossip/gossip_wire.csv +++ b/lightningd/gossip/gossip_wire.csv @@ -9,7 +9,7 @@ gossipstatus_fdpass_failed,0x8004 gossipstatus_peer_bad_msg,1000 gossipstatus_peer_bad_msg,0,unique_id,8 gossipstatus_peer_bad_msg,8,len,2 -gossipstatus_peer_bad_msg,10,err,len,u8 +gossipstatus_peer_bad_msg,10,err,len*u8 #include @@ -17,7 +17,7 @@ gossipstatus_peer_bad_msg,10,err,len,u8 # (if it is to move onto a channel, we get a status msg). gossipctl_new_peer,1 gossipctl_new_peer,0,unique_id,8 -gossipctl_new_peer,8,crypto_state,144,struct crypto_state +gossipctl_new_peer,8,crypto_state,struct crypto_state # Tell it to release a peer which has initialized. gossipctl_release_peer,2 @@ -26,7 +26,7 @@ gossipctl_release_peer,0,unique_id,8 # This releases the peer and returns the cryptostate (followed by fd) gossipctl_release_peer_reply,102 gossipctl_release_peer_reply,0,unique_id,8 -gossipctl_release_peer_reply,8,crypto_state,144,struct crypto_state +gossipctl_release_peer_reply,8,crypto_state,struct crypto_state # This is where we save a peer's features. #gossipstatus_peer_features,1 @@ -43,13 +43,13 @@ gossipstatus_peer_ready,0,unique_id,8 # Peer can send non-gossip packet (usually an open_channel) (followed by fd) gossipstatus_peer_nongossip,4 gossipstatus_peer_nongossip,0,unique_id,8 -gossipstatus_peer_nongossip,10,crypto_state,144,struct crypto_state +gossipstatus_peer_nongossip,10,crypto_state,struct crypto_state gossipstatus_peer_nongossip,154,len,2 -gossipstatus_peer_nongossip,156,msg,len,u8 +gossipstatus_peer_nongossip,156,msg,len*u8 # Pass JSON-RPC getnodes call through gossip_getnodes_request,5 gossip_getnodes_reply,105 -gossip_getnodes_reply,0,replen,2,u16 -gossip_getnodes_reply,2,reply,replen,u8 +gossip_getnodes_reply,0,replen,u16 +gossip_getnodes_reply,2,reply,replen*u8 diff --git a/lightningd/handshake/handshake_wire.csv b/lightningd/handshake/handshake_wire.csv index 2a46dee57..8937a26bb 100644 --- a/lightningd/handshake/handshake_wire.csv +++ b/lightningd/handshake/handshake_wire.csv @@ -36,12 +36,12 @@ handshake_responder,1 handshake_responder,1,my_id,33 handshake_responder_reply,101 handshake_responder_reply,0,initiator_id,33 -handshake_responder_reply,33,cs,144,struct crypto_state +handshake_responder_reply,33,cs,struct crypto_state handshake_initiator,2 handshake_initiator,0,my_id,33 handshake_initiator,33,responder_id,33 handshake_initiator_reply,102 -handshake_initiator_reply,0,cs,144,struct crypto_state +handshake_initiator_reply,0,cs,struct crypto_state diff --git a/lightningd/hsm/hsm_wire.csv b/lightningd/hsm/hsm_wire.csv index be25d8e6b..5294eeb16 100644 --- a/lightningd/hsm/hsm_wire.csv +++ b/lightningd/hsm/hsm_wire.csv @@ -9,17 +9,17 @@ hsmstatus_key_failed,0x8004 hsmstatus_client_bad_request,1000 hsmstatus_client_bad_request,0,unique-id,8 hsmstatus_client_bad_request,8,len,2 -hsmstatus_client_bad_request,10,msg,len,u8 +hsmstatus_client_bad_request,10,msg,len*u8 # Start the HSM. hsmctl_init,1 -hsmctl_init,0,new,1,bool +hsmctl_init,0,new,bool hsmctl_init_reply,101 hsmctl_init_reply,0,node_id,33 -hsmctl_init_reply,33,peer_seed,32,struct privkey +hsmctl_init_reply,33,peer_seed,struct privkey hsmctl_init_reply,65,bip32_len,2 -hsmctl_init_reply,67,bip32_seed,bip32_len*1,u8 +hsmctl_init_reply,67,bip32_seed,bip32_len*u8 # ECDH returns an fd. hsmctl_hsmfd_ecdh,3 @@ -38,10 +38,10 @@ hsmctl_sign_funding,16,change_keyindex,4 hsmctl_sign_funding,20,our_pubkey,33 hsmctl_sign_funding,52,their_pubkey,33 hsmctl_sign_funding,85,num_inputs,2 -hsmctl_sign_funding,87,inputs,num_inputs*49,struct utxo +hsmctl_sign_funding,87,inputs,num_inputs*struct utxo hsmctl_sign_funding_reply,104 hsmctl_sign_funding_reply,0,num_sigs,2 -hsmctl_sign_funding_reply,0,sig,num_sigs*64,secp256k1_ecdsa_signature +hsmctl_sign_funding_reply,0,sig,num_sigs*secp256k1_ecdsa_signature diff --git a/lightningd/opening/opening_wire.csv b/lightningd/opening/opening_wire.csv index ea7280e63..11b51a2f4 100644 --- a/lightningd/opening/opening_wire.csv +++ b/lightningd/opening/opening_wire.csv @@ -15,13 +15,13 @@ opening_peer_bad_initial_message,0x8014 #include opening_init,0 # Base configuration we'll offer (channel reserve will vary with amount) -opening_init,0,our_config,36,struct channel_config +opening_init,0,our_config,struct channel_config # Minimum/maximum configuration values we'll accept opening_init,36,max_to_self_delay,4 opening_init,40,min_effective_htlc_capacity_msat,8 -opening_init,48,crypto_state,144,struct crypto_state +opening_init,48,crypto_state,struct crypto_state # Seed to generate all the keys from -opening_init,196,seed,32,struct privkey +opening_init,196,seed,struct privkey # This means we offer the open. opening_open,1 @@ -37,14 +37,14 @@ opening_open_reply,0,remote_fundingkey,33 # Now we give the funding txid and outnum. opening_open_funding,2 -opening_open_funding,0,txid,32,struct sha256_double -opening_open_funding,32,txout,2,u16 +opening_open_funding,0,txid,struct sha256_double +opening_open_funding,32,txout,u16 # This gives their sig, means we can broadcast tx: we're done. opening_open_funding_reply,102 -opening_open_funding_reply,0,their_config,36,struct channel_config -opening_open_funding_reply,36,first_commit_sig,64,secp256k1_ecdsa_signature -opening_open_funding_reply,100,crypto_state,144,struct crypto_state +opening_open_funding_reply,0,their_config,struct channel_config +opening_open_funding_reply,36,first_commit_sig,secp256k1_ecdsa_signature +opening_open_funding_reply,100,crypto_state,struct crypto_state opening_open_funding_reply,244,revocation_basepoint,33 opening_open_funding_reply,277,payment_basepoint,33 opening_open_funding_reply,310,delayed_payment_basepoint,33 @@ -55,20 +55,20 @@ opening_accept,3 opening_accept,0,min_feerate,4 opening_accept,4,max_feerate,4 opening_accept,8,len,2 -opening_accept,10,msg,len,u8 +opening_accept,10,msg,len*u8 # This gives the txid of their funding tx to watch. opening_accept_reply,103 -opening_accept_reply,0,funding_txid,32,struct sha256_double +opening_accept_reply,0,funding_txid,struct sha256_double # Acknowledge watch is in place, now can send sig. opening_accept_finish,4 opening_accept_finish_reply,104 -opening_accept_finish_reply,32,funding_txout,2,u16 -opening_accept_finish_reply,0,their_config,36,struct channel_config -opening_accept_finish_reply,36,first_commit_sig,64,secp256k1_ecdsa_signature -opening_accept_finish_reply,100,crypto_state,144,struct crypto_state +opening_accept_finish_reply,32,funding_txout,u16 +opening_accept_finish_reply,0,their_config,struct channel_config +opening_accept_finish_reply,36,first_commit_sig,secp256k1_ecdsa_signature +opening_accept_finish_reply,100,crypto_state,struct crypto_state opening_accept_finish_reply,244,remote_fundingkey,33 opening_accept_finish_reply,277,revocation_basepoint,33 opening_accept_finish_reply,310,payment_basepoint,33 diff --git a/tools/generate-wire.py b/tools/generate-wire.py index ac7dc4431..f92961dc1 100755 --- a/tools/generate-wire.py +++ b/tools/generate-wire.py @@ -81,40 +81,48 @@ sizetypemap = { 1: FieldType('u8') } +# It would be nicer if we had put '*1' in spec and disallowed bare lenvar. +# In practice we only recognize raw lenvar when it's the previous field. + +# size := baresize | arraysize +# baresize := simplesize | lenvar +# simplesize := number | type +# arraysize := lenvar '*' simplesize class Field(object): - def __init__(self, message, name, size, comments, typename=None): + def __init__(self, message, name, size, comments, prevname): self.message = message self.comments = comments self.name = name.replace('-', '_') self.is_len_var = False self.lenvar = None + self.num_elems = 1 - # Size could be a literal number (eg. 33), or a field (eg 'len'), or - # a multiplier of a field (eg. num-htlc-timeouts*64). - try: - base_size = int(size) - except ValueError: - # If it's a multiplicitive expression, must end in basesize. - if '*' in size: - base_size = int(size.split('*')[1]) - self.lenvar = size.split('*')[0] - else: - base_size = 0 - self.lenvar = size - self.lenvar = self.lenvar.replace('-','_') - - if typename is None: - self.fieldtype = Field._guess_type(message,self.name,base_size) + # If it's an arraysize, swallow prefix. + if '*' in size: + self.lenvar = size.split('*')[0].replace('-','_') + size = size.split('*')[1] else: - self.fieldtype = FieldType(typename) + if size == prevname: + # Raw length field, implies u8. + self.lenvar = size.replace('-','_') + size = 'u8' - # Unknown types are assumed to have base_size: div by 0 if that's unknown. - if self.fieldtype.tsize == 0: - self.fieldtype.tsize = base_size + try: + # Just a number? Guess based on size. + base_size = int(size) + self.fieldtype = Field._guess_type(message,self.name,base_size) + # There are some arrays which we have to guess, based on sizes. + if base_size % self.fieldtype.tsize != 0: + raise ValueError('Invalid size {} for {}.{} not a multiple of {}' + .format(base_size, + self.message, + self.name, + self.fieldtype.tsize)) + self.num_elems = int(base_size / self.fieldtype.tsize) - if base_size % self.fieldtype.tsize != 0: - raise ValueError('Invalid size {} for {}.{} not a multiple of {}'.format(base_size,self.message,self.name,self.fieldtype.tsize)) - self.num_elems = int(base_size / self.fieldtype.tsize) + except ValueError: + # Not a number; must be a type. + self.fieldtype = FieldType(size) def is_padding(self): return self.name.startswith('pad') @@ -360,6 +368,7 @@ options = parser.parse_args() messages = [] comments = [] includes = [] +prevfield = None # Read csv lines. Single comma is the message values, more is offset/len. for line in fileinput.input(options.files): @@ -382,18 +391,19 @@ for line in fileinput.input(options.files): # eg commit_sig,132 messages.append(Message(parts[0],Enumtype("WIRE_" + parts[0].upper(), parts[1]), comments)) comments=[] - else: + prevfield = None + elif len(parts) == 4: # eg commit_sig,0,channel-id,8 OR - # commit_sig,0,channel-id,8,u64 + # commit_sig,0,channel-id,u64 for m in messages: if m.name == parts[0]: - if len(parts) == 4: - m.addField(Field(parts[0], parts[2], parts[3], comments)) - else: - m.addField(Field(parts[0], parts[2], parts[3], comments, - parts[4])) + m.addField(Field(parts[0], parts[2], parts[3], comments, prevfield)) + prevfield = parts[2] break comments=[] + else: + raise ValueError('Line {} malformed'.format(line.rstrip())) + header_template = """#ifndef LIGHTNING_{idem} #define LIGHTNING_{idem}