msggen: fix walk through nested json schemas

Changelog-None
This commit is contained in:
daywalker90 2024-04-14 17:18:11 +02:00 committed by Christian Decker
parent 507c2de9c1
commit ada4de1588
10 changed files with 1390 additions and 1067 deletions

View File

@ -28,6 +28,13 @@
"must-create": 0,
"must-replace": 1
},
"DecodeFallbacksType": {
"P2PKH": 0,
"P2SH": 1,
"P2TR": 4,
"P2WPKH": 2,
"P2WSH": 3
},
"DecodeType": {
"bolt11 invoice": 3,
"bolt12 invoice": 1,
@ -348,6 +355,17 @@
"StopResult": {
"Shutdown complete": 0
},
"WaitDetailsStatus": {
"complete": 5,
"expired": 2,
"failed": 4,
"local_failed": 8,
"offered": 6,
"paid": 1,
"pending": 3,
"settled": 7,
"unpaid": 0
},
"WaitIndexname": {
"created": 0,
"deleted": 2,
@ -515,6 +533,9 @@
"Decode.extra[].tag": 1
},
"DecodeFallbacks": {
"Decode.fallbacks[].addr": 3,
"Decode.fallbacks[].hex": 4,
"Decode.fallbacks[].type": 2,
"Decode.fallbacks[].warning_invoice_fallbacks_version_invalid": 1
},
"DecodeInvoice_fallbacks": {
@ -540,13 +561,17 @@
"Decode.string": 1
},
"DecodeResponse": {
"Decode.amount_msat": 79,
"Decode.created_at": 60,
"Decode.currency": 78,
"Decode.currency_minor_unit": 8,
"Decode.decrypted": 76,
"Decode.description": 80,
"Decode.description_hash": 64,
"Decode.expiry": 61,
"Decode.extra[]": 69,
"Decode.fallbacks[]": 59,
"Decode.features": 81,
"Decode.hex": 75,
"Decode.invoice_amount_msat": 44,
"Decode.invoice_created_at": 41,
@ -587,6 +612,7 @@
"Decode.payment_secret": 66,
"Decode.restrictions[]": 73,
"Decode.routes[][]": 68,
"Decode.signature": 77,
"Decode.string": 72,
"Decode.type": 1,
"Decode.unique_id": 70,
@ -674,6 +700,7 @@
},
"DelinvoiceResponse": {
"DelInvoice.amount_msat": 4,
"DelInvoice.amount_received_msat": 15,
"DelInvoice.bolt11": 2,
"DelInvoice.bolt12": 3,
"DelInvoice.created_index": 12,
@ -682,8 +709,11 @@
"DelInvoice.invreq_payer_note": 11,
"DelInvoice.label": 1,
"DelInvoice.local_offer_id": 9,
"DelInvoice.paid_at": 16,
"DelInvoice.pay_index": 14,
"DelInvoice.payer_note": 10,
"DelInvoice.payment_hash": 6,
"DelInvoice.payment_preimage": 17,
"DelInvoice.status": 7,
"DelInvoice.updated_index": 13
},
@ -838,6 +868,7 @@
"Getinfo.binding[].address": 2,
"Getinfo.binding[].port": 3,
"Getinfo.binding[].socket": 4,
"Getinfo.binding[].subtype": 5,
"Getinfo.binding[].type": 1
},
"GetinfoOur_features": {
@ -1014,6 +1045,8 @@
},
"ListforwardsForwards": {
"ListForwards.forwards[].created_index": 12,
"ListForwards.forwards[].failcode": 15,
"ListForwards.forwards[].failreason": 16,
"ListForwards.forwards[].fee_msat": 7,
"ListForwards.forwards[].in_channel": 1,
"ListForwards.forwards[].in_htlc_id": 10,
@ -1023,6 +1056,7 @@
"ListForwards.forwards[].out_msat": 8,
"ListForwards.forwards[].payment_hash": 6,
"ListForwards.forwards[].received_time": 4,
"ListForwards.forwards[].resolved_time": 14,
"ListForwards.forwards[].status": 3,
"ListForwards.forwards[].style": 9,
"ListForwards.forwards[].updated_index": 13
@ -1056,6 +1090,7 @@
"ListFunds.outputs[].output": 2,
"ListFunds.outputs[].redeemscript": 6,
"ListFunds.outputs[].reserved": 9,
"ListFunds.outputs[].reserved_to_block": 10,
"ListFunds.outputs[].scriptpubkey": 4,
"ListFunds.outputs[].status": 7,
"ListFunds.outputs[].txid": 1
@ -1124,13 +1159,22 @@
"ListNodes.nodes[].color": 4,
"ListNodes.nodes[].features": 5,
"ListNodes.nodes[].last_timestamp": 2,
"ListNodes.nodes[].nodeid": 1
"ListNodes.nodes[].nodeid": 1,
"ListNodes.nodes[].option_will_fund": 7
},
"ListnodesNodesAddresses": {
"ListNodes.nodes[].addresses[].address": 3,
"ListNodes.nodes[].addresses[].port": 2,
"ListNodes.nodes[].addresses[].type": 1
},
"ListnodesNodesOption_will_fund": {
"ListNodes.nodes[].option_will_fund.channel_fee_max_base_msat": 4,
"ListNodes.nodes[].option_will_fund.channel_fee_max_proportional_thousandths": 5,
"ListNodes.nodes[].option_will_fund.compact_lease": 6,
"ListNodes.nodes[].option_will_fund.funding_weight": 3,
"ListNodes.nodes[].option_will_fund.lease_fee_base_msat": 1,
"ListNodes.nodes[].option_will_fund.lease_fee_basis": 2
},
"ListnodesRequest": {
"ListNodes.id": 1
},
@ -1183,6 +1227,7 @@
"ListPeerChannels.channels[].close_to": 17,
"ListPeerChannels.channels[].close_to_addr": 53,
"ListPeerChannels.channels[].closer": 20,
"ListPeerChannels.channels[].direction": 60,
"ListPeerChannels.channels[].dust_limit_msat": 29,
"ListPeerChannels.channels[].features[]": 21,
"ListPeerChannels.channels[].fee_base_msat": 27,
@ -1201,6 +1246,7 @@
"ListPeerChannels.channels[].initial_feerate": 12,
"ListPeerChannels.channels[].last_feerate": 13,
"ListPeerChannels.channels[].last_stable_connection": 56,
"ListPeerChannels.channels[].last_tx_fee_msat": 59,
"ListPeerChannels.channels[].lost_state": 57,
"ListPeerChannels.channels[].max_accepted_htlcs": 40,
"ListPeerChannels.channels[].max_to_us_msat": 25,
@ -1842,6 +1888,20 @@
"UtxoPsbt.psbt": 1,
"UtxoPsbt.reservations[]": 6
},
"WaitDetails": {
"Wait.details.bolt11": 4,
"Wait.details.bolt12": 5,
"Wait.details.description": 3,
"Wait.details.groupid": 7,
"Wait.details.in_channel": 9,
"Wait.details.in_htlc_id": 10,
"Wait.details.in_msat": 11,
"Wait.details.label": 2,
"Wait.details.out_channel": 12,
"Wait.details.partid": 6,
"Wait.details.payment_hash": 8,
"Wait.details.status": 1
},
"WaitRequest": {
"Wait.indexname": 2,
"Wait.nextvalue": 3,
@ -1850,6 +1910,7 @@
"WaitResponse": {
"Wait.created": 2,
"Wait.deleted": 4,
"Wait.details": 5,
"Wait.subsystem": 1,
"Wait.updated": 3
},
@ -2307,10 +2368,18 @@
"added": "v23.05",
"deprecated": null
},
"Decode.amount_msat": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Decode.created_at": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Decode.currency": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Decode.currency_minor_unit": {
"added": "pre-v0.10.1",
"deprecated": false
@ -2319,6 +2388,10 @@
"added": "v23.11",
"deprecated": false
},
"Decode.description": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Decode.description_hash": {
"added": "pre-v0.10.1",
"deprecated": false
@ -2343,10 +2416,26 @@
"added": "pre-v0.10.1",
"deprecated": false
},
"Decode.fallbacks[].addr": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Decode.fallbacks[].hex": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Decode.fallbacks[].type": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Decode.fallbacks[].warning_invoice_fallbacks_version_invalid": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Decode.features": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Decode.hex": {
"added": "pre-v0.10.1",
"deprecated": false
@ -2647,6 +2736,10 @@
"added": "pre-v0.10.1",
"deprecated": false
},
"Decode.signature": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Decode.string": {
"added": "pre-v0.10.1",
"deprecated": false
@ -2947,6 +3040,10 @@
"added": "pre-v0.10.1",
"deprecated": false
},
"DelInvoice.amount_received_msat": {
"added": "pre-v0.10.1",
"deprecated": false
},
"DelInvoice.bolt11": {
"added": "pre-v0.10.1",
"deprecated": false
@ -2983,10 +3080,22 @@
"added": "pre-v0.10.1",
"deprecated": false
},
"DelInvoice.paid_at": {
"added": "pre-v0.10.1",
"deprecated": false
},
"DelInvoice.pay_index": {
"added": "pre-v0.10.1",
"deprecated": false
},
"DelInvoice.payment_hash": {
"added": "pre-v0.10.1",
"deprecated": false
},
"DelInvoice.payment_preimage": {
"added": "pre-v0.10.1",
"deprecated": false
},
"DelInvoice.status": {
"added": "pre-v0.10.1",
"deprecated": false
@ -3531,6 +3640,10 @@
"added": "pre-v0.10.1",
"deprecated": false
},
"Getinfo.binding[].subtype": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Getinfo.binding[].type": {
"added": "pre-v0.10.1",
"deprecated": false
@ -4007,6 +4120,14 @@
"added": "v23.11",
"deprecated": false
},
"ListForwards.forwards[].failcode": {
"added": "pre-v0.10.1",
"deprecated": false
},
"ListForwards.forwards[].failreason": {
"added": "pre-v0.10.1",
"deprecated": false
},
"ListForwards.forwards[].fee_msat": {
"added": "pre-v0.10.1",
"deprecated": false
@ -4039,6 +4160,10 @@
"added": "pre-v0.10.1",
"deprecated": false
},
"ListForwards.forwards[].resolved_time": {
"added": "pre-v0.10.1",
"deprecated": false
},
"ListForwards.forwards[].status": {
"added": "pre-v0.10.1",
"deprecated": false
@ -4147,6 +4272,10 @@
"added": "pre-v0.10.1",
"deprecated": false
},
"ListFunds.outputs[].reserved_to_block": {
"added": "pre-v0.10.1",
"deprecated": false
},
"ListFunds.outputs[].scriptpubkey": {
"added": "pre-v0.10.1",
"deprecated": false
@ -4363,6 +4492,34 @@
"added": "pre-v0.10.1",
"deprecated": false
},
"ListNodes.nodes[].option_will_fund": {
"added": "pre-v0.10.1",
"deprecated": false
},
"ListNodes.nodes[].option_will_fund.channel_fee_max_base_msat": {
"added": "pre-v0.10.1",
"deprecated": false
},
"ListNodes.nodes[].option_will_fund.channel_fee_max_proportional_thousandths": {
"added": "pre-v0.10.1",
"deprecated": false
},
"ListNodes.nodes[].option_will_fund.compact_lease": {
"added": "pre-v0.10.1",
"deprecated": false
},
"ListNodes.nodes[].option_will_fund.funding_weight": {
"added": "pre-v0.10.1",
"deprecated": false
},
"ListNodes.nodes[].option_will_fund.lease_fee_base_msat": {
"added": "pre-v0.10.1",
"deprecated": false
},
"ListNodes.nodes[].option_will_fund.lease_fee_basis": {
"added": "pre-v0.10.1",
"deprecated": false
},
"ListOffers": {
"added": "pre-v0.10.1",
"deprecated": null
@ -4535,6 +4692,10 @@
"added": "v23.02",
"deprecated": false
},
"ListPeerChannels.channels[].direction": {
"added": "v23.02",
"deprecated": false
},
"ListPeerChannels.channels[].dust_limit_msat": {
"added": "v23.02",
"deprecated": false
@ -4695,6 +4856,10 @@
"added": "v24.02",
"deprecated": false
},
"ListPeerChannels.channels[].last_tx_fee_msat": {
"added": "pre-v0.10.1",
"deprecated": false
},
"ListPeerChannels.channels[].lost_state": {
"added": "v24.02",
"deprecated": false
@ -6399,6 +6564,58 @@
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details.bolt11": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details.bolt12": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details.description": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details.groupid": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details.in_channel": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details.in_htlc_id": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details.in_msat": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details.label": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details.out_channel": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details.partid": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details.payment_hash": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.details.status": {
"added": "pre-v0.10.1",
"deprecated": false
},
"Wait.indexname": {
"added": "pre-v0.10.1",
"deprecated": false

Binary file not shown.

BIN
cln-grpc/src/convert.rs generated

Binary file not shown.

BIN
cln-rpc/src/model.rs generated

Binary file not shown.

View File

@ -34,7 +34,7 @@ typemap = {
"feerate": "Feerate",
"outputdesc": "OutputDesc",
"secret": "bytes",
"bip340sig": "bytes",
"bip340sig": "string",
"hash": "bytes",
}
@ -95,8 +95,10 @@ class GrpcGenerator(IGenerator):
def enumerate_fields(self, message_name, fields):
"""Use the meta map to identify which number this field will get.
"""
for f in fields:
yield (self.field2number(message_name, f), f)
enumerated_values = [(self.field2number(message_name, f), f) for f in fields]
sorted_enumerated_values = sorted(enumerated_values, key=lambda x: x[0])
for i, v in sorted_enumerated_values:
yield (i, v)
def enumvar2number(self, typename: TypeName, variant):
"""Find an existing variant number of generate a new one.
@ -122,8 +124,10 @@ class GrpcGenerator(IGenerator):
return m[typename][variant]
def enumerate_enum(self, typename, variants):
for v in variants:
yield (self.enumvar2number(typename, v), v)
enumerated_values = [(self.enumvar2number(typename, v), v) for v in variants]
sorted_enumerated_values = sorted(enumerated_values, key=lambda x: x[0])
for i, v in sorted_enumerated_values:
yield (i, v)
def gather_types(self, service):
"""Gather all types that might need to be defined.
@ -249,6 +253,8 @@ class GrpcConverterGenerator(IGenerator):
if field.omit():
return
field.sort()
# First pass: generate any sub-fields before we generate the
# top-level field itself.
for f in field.fields:
@ -325,7 +331,6 @@ class GrpcConverterGenerator(IGenerator):
'hash?': f'c.{name}.map(|v| <Sha256 as AsRef<[u8]>>::as_ref(&v).to_vec())',
'secret': f'c.{name}.to_vec()',
'secret?': f'c.{name}.map(|v| v.to_vec())',
'msat_or_any': f'Some(c.{name}.into())',
'msat_or_all': f'Some(c.{name}.into())',
'msat_or_all?': f'c.{name}.map(|o|o.into())',

View File

@ -97,6 +97,7 @@ class Grpc2PyGenerator(IGenerator):
self.converters[field.path] = "str(m.{{name}})"
def generate_composite(self, prefix, field: CompositeField):
field.sort()
if override.get(field.path, "") is None:
return
name = field.name.normalized()

View File

@ -35,7 +35,7 @@ typemap = {
'outputdesc': 'OutputDesc',
'hash': 'Sha256',
'secret': 'Secret',
'bip340sig': 'Secret',
'bip340sig': 'String',
'integer': 'i64',
}
@ -110,7 +110,8 @@ def gen_enum(e, meta):
complete_variants = False
if m != {} and complete_variants:
for v in e.variants:
sorted_variants = sorted(e.variants, key=lambda x: m[str(x)])
for v in sorted_variants:
if v is None:
continue
norm = v.normalized()
@ -128,7 +129,7 @@ def gen_enum(e, meta):
""")
if m != {} and complete_variants:
for v in e.variants:
for v in sorted_variants:
norm = v.normalized()
# decl += f" #[serde(rename = \"{v}\")]\n"
decl += f" {m[str(v)]} => Ok({e.typename}::{norm}),\n"
@ -241,6 +242,7 @@ def gen_composite(c, meta) -> Tuple[str, str]:
fields = []
for f in c.fields:
fields.append(gen_field(f, meta))
fields = sorted(fields)
r = "".join([f[1] for f in fields])

View File

@ -78,6 +78,15 @@ class Field:
self.type_override: Optional[str] = None
def __lt__(self, other):
return self.path < other.path
def __eq__(self, other):
return self.path == other.path
def __iter__(self):
yield self.path
@property
def name(self):
return FieldName(self.path.split(".")[-1])
@ -199,20 +208,45 @@ class CompositeField(Field):
'else': {'properties': js.get('else', {}).get('properties', [])},
}
# Yes, this is ugly, but walking nested dicts always is.
def merge_dicts(dict1, dict2):
merged_dict = {}
for key in set(dict1.keys()) | set(dict2.keys()):
if key in dict1 and key in dict2:
if isinstance(dict1[key], dict) and isinstance(dict2[key], dict):
merged_dict[key] = merge_dicts(dict1[key], dict2[key])
else:
if isinstance(dict1[key], list) and isinstance(dict2[key], list):
merged_dict[key] = sorted(list(set(dict1[key]).union(set(dict2[key]))))
elif key in dict1:
merged_dict[key] = dict1[key]
else:
merged_dict[key] = dict2[key]
elif key in dict1:
merged_dict[key] = dict1[key]
else:
merged_dict[key] = dict2[key]
return merged_dict
for a in [top] + js.get('allOf', []):
var = a.get('then', {})
props = var.get('properties', None)
props = var.get('properties', {})
if isinstance(props, dict):
for k, v in props.items():
if k not in properties:
properties[k] = v
if properties != {}:
if k in properties:
properties[k] = merge_dicts(properties[k], v)
else:
properties[k] = v
var = a.get('else', {})
props = var.get('properties', None)
props = var.get('properties', {})
if isinstance(props, dict):
for k, v in props.items():
if k not in properties:
properties[k] = v
if properties != {}:
if k in properties:
properties[k] = merge_dicts(properties[k], v)
else:
properties[k] = v
# Identify required fields
required = js.get("required", [])
fields = []
@ -262,6 +296,9 @@ class CompositeField(Field):
typename, fields, path, js["description"] if "description" in js else "", added=js.get('added', None), deprecated=js.get('deprecated', None)
)
def sort(self):
self.fields = sorted(self.fields)
def __str__(self):
fieldnames = ",".join([f.path.split(".")[-1] for f in self.fields])
return f"CompositeField[name={self.path}, fields=[{fieldnames}]]"
@ -276,6 +313,12 @@ class EnumVariant(Field):
def __str__(self):
return self.variant
def __lt__(self, other):
return self.variant < other.variant
def __eq__(self, other):
return self.variant == other.variant
def normalized(self):
return self.variant.replace(' ', '_').replace('-', '_').replace("/", "_").upper()

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff