diff --git a/tools/extract-formats.py b/tools/extract-formats.py index 8956561..2a83e74 100755 --- a/tools/extract-formats.py +++ b/tools/extract-formats.py @@ -1,19 +1,20 @@ #! /usr/bin/python3 # Simple script to parse specs and produce CSV files. -# Released by Rusty Russell under CC0: https://creativecommons.org/publicdomain/zero/1.0/ +# Released by Rusty Russell under CC0: +# https://creativecommons.org/publicdomain/zero/1.0/ from optparse import OptionParser import sys import re import fileinput -# Figure out if we can determine type from size. -def guess_alignment(message,name,sizestr): +# Figure out if we can determine type from size. +def guess_alignment(message, name, sizestr): # Exceptions: # - Padding has no alignment requirements. # - channel-id is size 8, but has alignment 4. - # - node_announcement.ipv6 has size 16, but alignment 4 (to align IPv4 addr). + # - node_announcement.ipv6 has size 16, but alignment 4 (to align IPv4). # - node_announcement.alias is a string, so alignment 1 # - signatures have no alignment requirement. if name.startswith('pad'): @@ -29,7 +30,7 @@ def guess_alignment(message,name,sizestr): return 1 if 'signature' in name: - return 1 + return 1 # Size can be variable. try: @@ -51,6 +52,7 @@ def guess_alignment(message,name,sizestr): return 1 + def main(options, args=None, output=sys.stdout, lines=None): # Example inputs: # 1. type: 17 (`error`) @@ -62,23 +64,28 @@ def main(options, args=None, output=sys.stdout, lines=None): # 1. type: PERM|NODE|3 (`required_node_feature_missing`) message = None havedata = None - typeline = re.compile('1\. type: (?P[-0-9A-Za-z_|]+) \(`(?P[A-Za-z_]+)`\)') - dataline = re.compile('\s+\* \[`(?P[_a-z0-9*+]+)`:`(?P[_a-z0-9]+)`\]') + typeline = re.compile( + '1\. type: (?P[-0-9A-Za-z_|]+) \(`(?P[A-Za-z_]+)`\)') + dataline = re.compile( + '\s+\* \[`(?P[_a-z0-9*+]+)`:`(?P[_a-z0-9]+)`\]') if lines is None: lines = fileinput.input(args) - for i,line in enumerate(lines): + for i, line in enumerate(lines): line = line.rstrip() linenum = i+1 match = typeline.fullmatch(line) if match: if message is not None: - raise ValueError('{}:Found a message while I was already in a message'.format(linenum)) + raise ValueError('{}:Found a message while I was already in a ' + 'message'.format(linenum)) message = match.group('name') if options.output_types: - print("{},{}".format(match.group('name'), match.group('value')), file=output) + print("{},{}".format( + match.group('name'), + match.group('value')), file=output) havedata = None alignoff = False elif message is not None and havedata is None: @@ -87,11 +94,11 @@ def main(options, args=None, output=sys.stdout, lines=None): havedata = True dataoff = 0 off_extraterms = "" - maxalign = 1 elif message is not None and havedata is not None: match = dataline.fullmatch(line) if match: - align = guess_alignment(message, match.group('name'), match.group('size')) + align = guess_alignment(message, match.group('name'), + match.group('size')) # Do not check alignment if we previously had a variable # length field in the message @@ -99,10 +106,21 @@ def main(options, args=None, output=sys.stdout, lines=None): alignoff = True if not alignoff and options.check_alignment and dataoff % align != 0: - raise ValueError('{}:message {} field {} Offset {} not aligned on {} boundary:'.format(linenum, message, match.group('name'), dataoff, align)) + raise ValueError('{}:message {} field {} Offset {} not ' + 'aligned on {} boundary:'.format( + linenum, + message, + match.group('name'), + dataoff, + align)) if options.output_fields: - print("{},{}{},{},{}".format(message,dataoff,off_extraterms,match.group('name'),match.group('size')), file=output) + print("{},{}{},{},{}".format( + message, + dataoff, + off_extraterms, + match.group('name'), + match.group('size')), file=output) # Size can be variable. try: @@ -113,17 +131,30 @@ def main(options, args=None, output=sys.stdout, lines=None): else: message = None -if __name__=="__main__": + +if __name__ == "__main__": parser = OptionParser() - parser.add_option("--message-types", - action="store_true", dest="output_types", default=False, - help="Output MESSAGENAME,VALUE for every message") - parser.add_option("--check-alignment", - action="store_true", dest="check_alignment", default=False, - help="Check alignment for every member of each message") - parser.add_option("--message-fields", - action="store_true", dest="output_fields", default=False, - help="Output MESSAGENAME,OFFSET,FIELDNAME,SIZE for every message") + parser.add_option( + "--message-types", + action="store_true", + dest="output_types", + default=False, + help="Output MESSAGENAME,VALUE for every message" + ) + parser.add_option( + "--check-alignment", + action="store_true", + dest="check_alignment", + default=False, + help="Check alignment for every member of each message" + ) + parser.add_option( + "--message-fields", + action="store_true", + dest="output_fields", + default=False, + help="Output MESSAGENAME,OFFSET,FIELDNAME,SIZE for every message" + ) (options, args) = parser.parse_args() diff --git a/tools/structured.py b/tools/structured.py index f95bda0..00573a2 100644 --- a/tools/structured.py +++ b/tools/structured.py @@ -1,24 +1,28 @@ -formats = __import__("extract-formats") from io import StringIO import glob import collections import json +formats = __import__("extract-formats") + + class Options(object): - output_types = True - output_fields = True - check_alignment = False + output_types = True + output_fields = True + check_alignment = False + options = Options() csv = [] output = StringIO() for i in sorted(glob.glob("../??-*.md")): - with open(i) as f: - formats.main(options, output=output, lines=f.readlines()) - csvstr = output.getvalue().strip() - if csvstr == "": continue - csv += csvstr.split("\n") + with open(i) as f: + formats.main(options, output=output, lines=f.readlines()) + csvstr = output.getvalue().strip() + if csvstr == "": + continue + csv += csvstr.split("\n") resmap = collections.OrderedDict() @@ -26,20 +30,21 @@ currentmsgname = None currentmsgfields = {} typenum = None for line in csv: - parts = line.split(",") - if len(parts) == 2: - if currentmsgname is not None: - resmap[currentmsgname] = collections.OrderedDict([("type", typenum), ("payload", currentmsgfields)]) - currentmsgfields = collections.OrderedDict() - currentmsgname = parts[0] - typenum = parts[1] - continue - assert currentmsgname == parts[0], line - assert len(parts) == 4, line - position = parts[1] - length = parts[3] - fieldname = parts[2] - currentmsgfields[fieldname] = {"position": position, "length": length} + parts = line.split(",") + if len(parts) == 2: + if currentmsgname is not None: + resmap[currentmsgname] = collections.OrderedDict( + [("type", typenum), ("payload", currentmsgfields)]) + currentmsgfields = collections.OrderedDict() + currentmsgname = parts[0] + typenum = parts[1] + continue + assert currentmsgname == parts[0], line + assert len(parts) == 4, line + position = parts[1] + length = parts[3] + fieldname = parts[2] + currentmsgfields[fieldname] = {"position": position, "length": length} if __name__ == "__main__": - print(json.dumps(resmap, indent=True)) + print(json.dumps(resmap, indent=True))