From fa448bd21e356b9b3b4d9f08e8757433932e4f8a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 23 Nov 2016 11:24:08 +1030 Subject: [PATCH 1/3] BOLT 1: clean up message formatting to match other fields. Make the parser's life trivial. Signed-off-by: Rusty Russell --- 01-messaging-crypto-and-init.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/01-messaging-crypto-and-init.md b/01-messaging-crypto-and-init.md index ba78b22..1a5e61a 100644 --- a/01-messaging-crypto-and-init.md +++ b/01-messaging-crypto-and-init.md @@ -682,8 +682,12 @@ features supported or required by this node. Odd features are optional, even features are compulsory ("it's OK to be odd!"). The meaning of these bits will be defined in future. -1. type: 16 (MSG_INIT) -2. data: [4:len] [len:globalfeatures] [4:len] [len:localfeatures] +1. type: 16 (`MSG_INIT`) +2. data: + * [4:gflen] + * [gflen:globalfeatures] + * [4:lflen] + * [lflen:localfeatures] The 4-byte len fields indicate the number of bytes in the immediately following field. @@ -727,8 +731,10 @@ something is incorrect. 1. type: 17 (`MSG_ERROR`) -2. data: [8:channel-id] [4:len] [len:data] - +2. data: + * [8:channel-id] + * [4:len] + * [len:data] The 4-byte len field indicates the number of bytes in the immediately following field. From b742d2f73d5f0faa8cdf216605107ac476dad101 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 23 Nov 2016 11:33:25 +1030 Subject: [PATCH 2/3] BOLT 1, BOLT 2, BOLT 7: Remove message enums The MSG_XXX is trivially derived from the name of the message if people want to do that, so just change it to the actual message name. Signed-off-by: Rusty Russell --- 01-messaging-crypto-and-init.md | 12 ++++++------ 02-peer-protocol.md | 26 +++++++++++++------------- 07-routing-gossip.md | 6 +++--- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/01-messaging-crypto-and-init.md b/01-messaging-crypto-and-init.md index 1a5e61a..2a3d4d4 100644 --- a/01-messaging-crypto-and-init.md +++ b/01-messaging-crypto-and-init.md @@ -682,7 +682,7 @@ features supported or required by this node. Odd features are optional, even features are compulsory ("it's OK to be odd!"). The meaning of these bits will be defined in future. -1. type: 16 (`MSG_INIT`) +1. type: 16 (`init`) 2. data: * [4:gflen] * [gflen:globalfeatures] @@ -707,7 +707,7 @@ The receiving node MUST fail the channels if it receives a not understand. -Each node MUST wait to receive MSG_INIT before sending any other +Each node MUST wait to receive `init` before sending any other messages. @@ -730,7 +730,7 @@ For simplicity of diagnosis, it is often useful to tell the peer that something is incorrect. -1. type: 17 (`MSG_ERROR`) +1. type: 17 (`error`) 2. data: * [8:channel-id] * [4:len] @@ -743,15 +743,15 @@ following field. ### Requirements -A node SHOULD send `MSG_ERROR` for protocol violations or internal +A node SHOULD send `error` for protocol violations or internal errors which make channels unusable or further communication unusable. -A node MAY send an empty [data] field. A node sending `MSG_ERROR` MUST +A node MAY send an empty [data] field. A node sending `error` MUST fail the channel referred to by the `channel-id`, or if `channel-id` is 0xFFFFFFFFFFFFFFFF it MUST fail all channels and MUST close the connection. A node MUST NOT set `len` to greater than the data length. -A node receiving `MSG_ERROR` MUST fail the channel referred to by +A node receiving `error` MUST fail the channel referred to by `channel-id`, or if `channel-id` is 0xFFFFFFFFFFFFFFFF it MUST fail all channels and MUST close the connection. A receiving node MUST truncate `len` to the remainder of the packet if it is larger. diff --git a/02-peer-protocol.md b/02-peer-protocol.md index 17e603a..22bf264 100644 --- a/02-peer-protocol.md +++ b/02-peer-protocol.md @@ -68,7 +68,7 @@ fails. This message contains information about a node, and indicates its desire to set up a new channel. -1. type: 32 (`MSG_OPEN_CHANNEL`) +1. type: 32 (`open_channel`) 2. data: * [8:temporary-channel-id] * [8:funding-satoshis] @@ -162,7 +162,7 @@ This message contains information about a node, and indicates its acceptance of the new channel. -1. type: 33 (`MSG_ACCEPT_CHANNEL`) +1. type: 33 (`accept_channel`) 2. data: * [8:temporary-channel-id] * [8:dust-limit-satoshis] @@ -193,7 +193,7 @@ This message describes the outpoint which the funder has created for the initial commitment transactions. After receiving the peer's signature, it will broadcast the funding transaction. -1. type: 34 (`MSG_FUNDING_CREATED`) +1. type: 34 (`funding_created`) 2. data: * [8:temporary-channel-id] * [32:txid] @@ -212,7 +212,7 @@ This message gives the funder the signature they need for the first commitment transaction, so they can broadcast it knowing they can redeem their funds if they need to. -1. type: 35 (`MSG_FUNDING_SIGNED`) +1. type: 35 (`funding_signed`) 2. data: * [8:temporary-channel-id] * [64:signature] @@ -227,7 +227,7 @@ The recipient MUST fail the channel if `signature` is incorrect. This message indicates that the funding transaction has reached the `minimum-depth` asked for in `accept_channel`. Once both nodes have sent this, the channel enters normal operating mode. -1. type: 36 (`MSG_FUNDING_LOCKED`) +1. type: 36 (`funding_locked`) 2. data: * [8:temporary-channel-id] * [8:channel-id] @@ -292,7 +292,7 @@ The exact calculation used for deriving the fee from the fee rate is given in [BOLT #3]. -1. type: 37 (`MSG_UPDATE_FEE`) +1. type: 37 (`update_fee`) 2. data: * [8:channel-id] * [4:feerate-per-kw] @@ -368,7 +368,7 @@ Either node (or both) can send a `shutdown` message to initiate closing, and indicating the scriptpubkey it wants to be paid to. -1. type: 38 (`MSG_SHUTDOWN`) +1. type: 38 (`shutdown`) 2. data: * [8:channel-id] * [4:len] @@ -421,7 +421,7 @@ signs the close transaction with the `script_pubkey` fields from the process terminates when both agree on the same fee, or one side fails the channel. -1. type: 39 (`MSG_CLOSING_SIGNED`) +1. type: 39 (`closing_signed`) 2. data: * [8:channel-id] * [8:fee-satoshis] @@ -551,7 +551,7 @@ The format of the `route` portion, which indicates where the payment is destined, is described in [BOLT #4]. -1. type: 128 (`MSG_UPDATE_ADD_HTLC`) +1. type: 128 (`update_add_htlc`) 2. data: * [8:channel-id] * [8:id] @@ -618,7 +618,7 @@ failed to route, or the payment preimage is supplied. The `reason` field is an opaque encrypted blob for the benefit of the original HTLC initiator as defined in [BOLT #4]. -1. type: 130 (`MSG_UPDATE_FULFILL_HTLC`) +1. type: 130 (`update_fulfill_htlc`) 2. data: * [8:channel-id] * [8:id] @@ -626,7 +626,7 @@ original HTLC initiator as defined in [BOLT #4]. For a timed out or route-failed HTLC: -1. type: 131 (`MSG_UPDATE_FAIL_HTLC`) +1. type: 131 (`update_fail_htlc`) 2. data: * [8:channel-id] * [8:id] @@ -665,7 +665,7 @@ sign the resulting transaction as defined in [BOLT #3] and send a `commitsig` message. -1. type: 132 (`MSG_COMMIT_SIG`) +1. type: 132 (`commit_sig`) 2. data: * [8:channel-id] * [64:signature] @@ -720,7 +720,7 @@ This message also supplies the signatures for the sender's HTLC-timeout transact The description of key derivation is in [BOLT #3](03-transactions.md#key-derivation). -1. type: 133 (`MSG_REVOKE_AND_ACK`) +1. type: 133 (`revoke_and_ack`) 2. data: * [8:channel-id] * [32:per-commitment-secret] diff --git a/07-routing-gossip.md b/07-routing-gossip.md index 82d397d..3162779 100644 --- a/07-routing-gossip.md +++ b/07-routing-gossip.md @@ -21,7 +21,7 @@ on-chain bitcoin key to the lightning node key, and vice-versa. The channel is not really usable until at least one side has announced its fee levels and expiry using `channel_update`. -1. type: 256 (`MSG_CHANNEL_ANNOUNCEMENT`) +1. type: 256 (`channel_announcement`) 2. data: * [64:node-signature-1] * [64:node-signature-2] @@ -104,7 +104,7 @@ This allows a node to indicate extra data associated with it, in addition to its public key. To avoid trivial denial of service attacks, nodes for which a channel is not already known are ignored. -1. type: 257 (`MSG_NODE_ANNOUNCEMENT`) +1. type: 257 (`node_announcement`) 2. data: * [64:signature] * [4:timestamp] @@ -166,7 +166,7 @@ channel shortid which matches the `channel_announcement` and one byte to indicate which end this is. It can do this multiple times, if it wants to change fees. -1. type: 258 (`MSG_CHANNEL_UPDATE`) +1. type: 258 (`channel_update`) 2. data: * [64:signature] * [4:blockheight] From fab307b57dd0553dfe5949c994ad7f77c2a35e83 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 23 Nov 2016 11:54:14 +1030 Subject: [PATCH 3/3] tools/extract-formats.py: extract message definitions and check alignment. Signed-off-by: Rusty Russell --- tools/extract-formats.py | 115 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100755 tools/extract-formats.py diff --git a/tools/extract-formats.py b/tools/extract-formats.py new file mode 100755 index 0000000..75fe8f2 --- /dev/null +++ b/tools/extract-formats.py @@ -0,0 +1,115 @@ +#! /usr/bin/python3 +# Simple script to parse specs and produce CSV files. +# Released by Rusty Russell under CC0: https://creativecommons.org/publicdomain/zero/1.0/ + +from optparse import OptionParser +import sys +import re +import fileinput + +# Figure out if we can determine type from size. +def guess_alignment(message,name,sizestr): + + # Exceptions: + # - Padding has no alignment requirements. + # - channel-id is size 8, but has alignment 4. + # - node_announcement.ipv6 has size 16, but alignment 4 (to align IPv4 addr). + # - node_announcement.alias is a string, so alignment 1 + # - signatures have alignment 4, because why not. + if match.group('name').startswith('pad'): + return 1 + + if match.group('name') == 'channel-id': + return 4 + + if message == 'node_announcement' and match.group('name') == 'ipv6': + return 4 + + if message == 'node_announcement' and match.group('name') == 'alias': + return 1 + + if 'signature' in match.group('name'): + return 4 + + # Size can be variable. + try: + size = int(match.group('size')) + except ValueError: + # If it contains a "*xxx" factor, that's our per-unit size. + s = re.search('\*([0-9]*)$', match.group('size')) + if s is None: + size = 1 + else: + size = int(s.group(1)) + + if size % 8 == 0: + return 8 + elif size % 4 == 0: + return 4 + elif size % 2 == 0: + return 2 + + return 1 + +parser = OptionParser() +parser.add_option("--message-types", + action="store_true", dest="output_types", default=False, + help="Output MESSAGENAME,VALUE for every message") +parser.add_option("--check-alignment", + action="store_true", dest="check_alignment", default=False, + help="Check alignment for every member of each message") +parser.add_option("--message-fields", + action="store_true", dest="output_fields", default=False, + help="Output MESSAGENAME,OFFSET,FIELDNAME,SIZE for every message") + +(options, args) = parser.parse_args() + +# Example input: +# 1. type: 17 (`MSG_ERROR`) +# 2. data: +# * [8:channel-id] +# * [4:len] +# * [len:data] +message = None +havedata = None +typeline = re.compile('1\. type: (?P[0-9]+) \(`(?P[-A-Za-z_]+)`\)') +dataline = re.compile('\s+\* \[(?P[-a-z0-9*+]+):(?P[-a-z0-9]+)\]') + +for i,line in enumerate(fileinput.input(args)): + line = line.rstrip() + linenum = i+1 + + match = typeline.fullmatch(line) + if match: + if message is not None: + raise ValueError('{}:Found a message while I was already in a message'.format(linenum)) + message = match.group('name') + if options.output_types: + print("{},{}".format(match.group('name'), match.group('value'))) + havedata = None + elif message is not None and havedata is None: + if line != '2. data:': + raise ValueError('{}:Expected 2. data:'.format(linenum)) + havedata = True + dataoff = 0 + off_extraterms = "" + maxalign = 1 + elif message is not None and havedata is not None: + match = dataline.fullmatch(line) + if match: + align = guess_alignment(message, match.group('name'), match.group('size')) + + if options.check_alignment and dataoff % align != 0: + raise ValueError('{}:message {} field {} Offset {} not aligned on {} boundary:'.format(linenum, message, match.group('name'), dataoff, align)) + + if options.output_fields: + print("{},{}{},{},{}".format(message,dataoff,off_extraterms,match.group('name'),match.group('size'))) + + # Size can be variable. + try: + dataoff += int(match.group('size')) + except ValueError: + # Offset has variable component. + off_extraterms = off_extraterms + "+" + match.group('size') + else: + message = None