From f72b0fdabd59741f15252080a0c00c0f22b43f51 Mon Sep 17 00:00:00 2001 From: ShahanaFarooqui Date: Thu, 22 Feb 2024 21:45:08 -0800 Subject: [PATCH] docs: Updates script and fixed test to generate lightning-sql - Updated doc/Makefile for generating schemas/lightning-sql.json - Read the lightning-sql template from schemas/lightning-sql-template.json - Generate sql tables data and merge it with json object read from above template - Save final merged object in schemas/lightning-sql.json - Updated doc/Makefile to auto generate lightning-sql.7.md and lightning-sql.7 - Deleted schemas/lightning-sql.json and adding it into .gitignore - Added lightning-sql specific titles in the fromschema.py script - Fixed test_sql by changing the sequence for listpeerchannels `reestablished` field - Ignoring lightning-sql.json for msggen schema.json because it is auto generated by sql plugin and lightning-sql-template.json. --- .gitignore | 1 + contrib/msggen/msggen/schema.json | 5 +- contrib/msggen/msggen/utils/utils.py | 3 +- doc/Makefile | 11 +- doc/lightning-sql.7.md | 382 ++++++++++++++++++++++++ doc/schemas/lightning-sql-template.json | 265 ++++++++++++++++ doc/schemas/lightning-sql.json | 266 ----------------- tests/test_plugin.py | 6 +- tools/fromschema.py | 2 +- 9 files changed, 662 insertions(+), 279 deletions(-) create mode 100644 doc/schemas/lightning-sql-template.json delete mode 100644 doc/schemas/lightning-sql.json diff --git a/.gitignore b/.gitignore index 8d2e96ec6..5f7cbef5a 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ tests/plugins/test_selfdisable_after_getmanifest # Ignore generated files devtools/features +doc/schemas/lightning-sql.json doc/lightning*.[1578] doc/reckless*.[1578] *_sqlgen.[ch] diff --git a/contrib/msggen/msggen/schema.json b/contrib/msggen/msggen/schema.json index fe695da2e..108dc6d78 100644 --- a/contrib/msggen/msggen/schema.json +++ b/contrib/msggen/msggen/schema.json @@ -29266,7 +29266,7 @@ "Main web site: " ] }, - "lightning-sql.json": { + "lightning-sql-template.json": { "$schema": "../rpc-schema-draft.json", "type": "object", "additionalProperties": false, @@ -29369,8 +29369,7 @@ ], "tables": [ "Note that the first column of every table is a unique integer called `rowid`: this is used for related tables to refer to specific rows in their parent. sqlite3 usually has this as an implicit column, but we make it explicit as the implicit version is not allowed to be used as a foreign key.", - "", - "The following tables are currently supported:\n- `bkpr_accountevents` (see lightning-bkpr-listaccountevents(7))\n - `account` (type `string`, sqltype `TEXT`)\n - `type` (type `string`, sqltype `TEXT`)\n - `tag` (type `string`, sqltype `TEXT`)\n - `credit_msat` (type `msat`, sqltype `INTEGER`)\n - `debit_msat` (type `msat`, sqltype `INTEGER`)\n - `currency` (type `string`, sqltype `TEXT`)\n - `timestamp` (type `u32`, sqltype `INTEGER`)\n - `outpoint` (type `string`, sqltype `TEXT`)\n - `blockheight` (type `u32`, sqltype `INTEGER`)\n - `origin` (type `string`, sqltype `TEXT`)\n - `payment_id` (type `hex`, sqltype `BLOB`)\n - `txid` (type `txid`, sqltype `BLOB`)\n - `description` (type `string`, sqltype `TEXT`)\n - `fees_msat` (type `msat`, sqltype `INTEGER`)\n - `is_rebalance` (type `boolean`, sqltype `INTEGER`)\n - `part_id` (type `u32`, sqltype `INTEGER`)\n\n- `bkpr_income` (see lightning-bkpr-listincome(7))\n - `account` (type `string`, sqltype `TEXT`)\n - `tag` (type `string`, sqltype `TEXT`)\n - `credit_msat` (type `msat`, sqltype `INTEGER`)\n - `debit_msat` (type `msat`, sqltype `INTEGER`)\n - `currency` (type `string`, sqltype `TEXT`)\n - `timestamp` (type `u32`, sqltype `INTEGER`)\n - `description` (type `string`, sqltype `TEXT`)\n - `outpoint` (type `string`, sqltype `TEXT`)\n - `txid` (type `txid`, sqltype `BLOB`)\n - `payment_id` (type `hex`, sqltype `BLOB`)\n\n- `channels` indexed by `short_channel_id` (see lightning-listchannels(7))\n - `source` (type `pubkey`, sqltype `BLOB`)\n - `destination` (type `pubkey`, sqltype `BLOB`)\n - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`)\n - `direction` (type `u32`, sqltype `INTEGER`)\n - `public` (type `boolean`, sqltype `INTEGER`)\n - `amount_msat` (type `msat`, sqltype `INTEGER`)\n - `message_flags` (type `u8`, sqltype `INTEGER`)\n - `channel_flags` (type `u8`, sqltype `INTEGER`)\n - `active` (type `boolean`, sqltype `INTEGER`)\n - `last_update` (type `u32`, sqltype `INTEGER`)\n - `base_fee_millisatoshi` (type `u32`, sqltype `INTEGER`)\n - `fee_per_millionth` (type `u32`, sqltype `INTEGER`)\n - `delay` (type `u32`, sqltype `INTEGER`)\n - `htlc_minimum_msat` (type `msat`, sqltype `INTEGER`)\n - `htlc_maximum_msat` (type `msat`, sqltype `INTEGER`)\n - `features` (type `hex`, sqltype `BLOB`)\n\n- `closedchannels` (see lightning-listclosedchannels(7))\n - `peer_id` (type `pubkey`, sqltype `BLOB`)\n - `channel_id` (type `hash`, sqltype `BLOB`)\n - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`)\n - `alias_local` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`)\n - `alias_remote` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`)\n - `opener` (type `string`, sqltype `TEXT`)\n - `closer` (type `string`, sqltype `TEXT`)\n - `private` (type `boolean`, sqltype `INTEGER`)\n - related table `closedchannels_channel_type_bits`, from JSON object `channel_type`\n - `row` (reference to `closedchannels_channel_type.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `bits` (type `u32`, sqltype `INTEGER`)\n - related table `closedchannels_channel_type_names`, from JSON object `channel_type`\n - `row` (reference to `closedchannels_channel_type.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `names` (type `string`, sqltype `TEXT`)\n - `total_local_commitments` (type `u64`, sqltype `INTEGER`)\n - `total_remote_commitments` (type `u64`, sqltype `INTEGER`)\n - `total_htlcs_sent` (type `u64`, sqltype `INTEGER`)\n - `funding_txid` (type `txid`, sqltype `BLOB`)\n - `funding_outnum` (type `u32`, sqltype `INTEGER`)\n - `leased` (type `boolean`, sqltype `INTEGER`)\n - `funding_fee_paid_msat` (type `msat`, sqltype `INTEGER`)\n - `funding_fee_rcvd_msat` (type `msat`, sqltype `INTEGER`)\n - `funding_pushed_msat` (type `msat`, sqltype `INTEGER`)\n - `total_msat` (type `msat`, sqltype `INTEGER`)\n - `final_to_us_msat` (type `msat`, sqltype `INTEGER`)\n - `min_to_us_msat` (type `msat`, sqltype `INTEGER`)\n - `max_to_us_msat` (type `msat`, sqltype `INTEGER`)\n - `last_commitment_txid` (type `hash`, sqltype `BLOB`)\n - `last_commitment_fee_msat` (type `msat`, sqltype `INTEGER`)\n - `close_cause` (type `string`, sqltype `TEXT`)\n - `last_stable_connection` (type `u64`, sqltype `INTEGER`)\n\n- `forwards` indexed by `in_channel and in_htlc_id` (see lightning-listforwards(7))\n - `created_index` (type `u64`, sqltype `INTEGER`)\n - `in_channel` (type `short_channel_id`, sqltype `TEXT`)\n - `in_htlc_id` (type `u64`, sqltype `INTEGER`)\n - `in_msat` (type `msat`, sqltype `INTEGER`)\n - `status` (type `string`, sqltype `TEXT`)\n - `received_time` (type `number`, sqltype `REAL`)\n - `out_channel` (type `short_channel_id`, sqltype `TEXT`)\n - `out_htlc_id` (type `u64`, sqltype `INTEGER`)\n - `updated_index` (type `u64`, sqltype `INTEGER`)\n - `style` (type `string`, sqltype `TEXT`)\n - `fee_msat` (type `msat`, sqltype `INTEGER`)\n - `out_msat` (type `msat`, sqltype `INTEGER`)\n - `resolved_time` (type `number`, sqltype `REAL`)\n - `failcode` (type `u32`, sqltype `INTEGER`)\n - `failreason` (type `string`, sqltype `TEXT`)\n\n- `htlcs` indexed by `short_channel_id and id` (see lightning-listhtlcs(7))\n - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`)\n - `id` (type `u64`, sqltype `INTEGER`)\n - `expiry` (type `u32`, sqltype `INTEGER`)\n - `amount_msat` (type `msat`, sqltype `INTEGER`)\n - `direction` (type `string`, sqltype `TEXT`)\n - `payment_hash` (type `hash`, sqltype `BLOB`)\n - `state` (type `string`, sqltype `TEXT`)\n\n- `invoices` indexed by `payment_hash` (see lightning-listinvoices(7))\n - `label` (type `string`, sqltype `TEXT`)\n - `description` (type `string`, sqltype `TEXT`)\n - `payment_hash` (type `hash`, sqltype `BLOB`)\n - `status` (type `string`, sqltype `TEXT`)\n - `expires_at` (type `u64`, sqltype `INTEGER`)\n - `amount_msat` (type `msat`, sqltype `INTEGER`)\n - `bolt11` (type `string`, sqltype `TEXT`)\n - `bolt12` (type `string`, sqltype `TEXT`)\n - `local_offer_id` (type `hash`, sqltype `BLOB`)\n - `invreq_payer_note` (type `string`, sqltype `TEXT`)\n - `created_index` (type `u64`, sqltype `INTEGER`)\n - `updated_index` (type `u64`, sqltype `INTEGER`)\n - `pay_index` (type `u64`, sqltype `INTEGER`)\n - `amount_received_msat` (type `msat`, sqltype `INTEGER`)\n - `paid_at` (type `u64`, sqltype `INTEGER`)\n - `paid_outpoint_txid` (type `txid`, sqltype `BLOB`, from JSON object `paid_outpoint`)\n - `paid_outpoint_outnum` (type `u32`, sqltype `INTEGER`, from JSON object `paid_outpoint`)\n - `payment_preimage` (type `secret`, sqltype `BLOB`)\n\n- `nodes` indexed by `nodeid` (see lightning-listnodes(7))\n - `nodeid` (type `pubkey`, sqltype `BLOB`)\n - `last_timestamp` (type `u32`, sqltype `INTEGER`)\n - `alias` (type `string`, sqltype `TEXT`)\n - `color` (type `hex`, sqltype `BLOB`)\n - `features` (type `hex`, sqltype `BLOB`)\n - related table `nodes_addresses`\n - `row` (reference to `nodes.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `type` (type `string`, sqltype `TEXT`)\n - `port` (type `u16`, sqltype `INTEGER`)\n - `address` (type `string`, sqltype `TEXT`)\n - `option_will_fund_lease_fee_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `option_will_fund`)\n - `option_will_fund_lease_fee_basis` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`)\n - `option_will_fund_funding_weight` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`)\n - `option_will_fund_channel_fee_max_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `option_will_fund`)\n - `option_will_fund_channel_fee_max_proportional_thousandths` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`)\n - `option_will_fund_compact_lease` (type `hex`, sqltype `BLOB`, from JSON object `option_will_fund`)\n\n- `offers` indexed by `offer_id` (see lightning-listoffers(7))\n - `offer_id` (type `hash`, sqltype `BLOB`)\n - `active` (type `boolean`, sqltype `INTEGER`)\n - `single_use` (type `boolean`, sqltype `INTEGER`)\n - `bolt12` (type `string`, sqltype `TEXT`)\n - `used` (type `boolean`, sqltype `INTEGER`)\n - `label` (type `string`, sqltype `TEXT`)\n\n- `peerchannels` indexed by `peer_id` (see lightning-listpeerchannels(7))\n - `peer_id` (type `pubkey`, sqltype `BLOB`)\n - `peer_connected` (type `boolean`, sqltype `INTEGER`)\n - `reestablished` (type `boolean`, sqltype `INTEGER`)\n - `state` (type `string`, sqltype `TEXT`)\n - `scratch_txid` (type `txid`, sqltype `BLOB`)\n - related table `peerchannels_channel_type_bits`, from JSON object `channel_type`\n - `row` (reference to `peerchannels_channel_type.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `bits` (type `u32`, sqltype `INTEGER`)\n - related table `peerchannels_channel_type_names`, from JSON object `channel_type`\n - `row` (reference to `peerchannels_channel_type.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `names` (type `string`, sqltype `TEXT`)\n - `local_htlc_minimum_msat` (type `msat`, sqltype `INTEGER`, from JSON object `local`)\n - `local_htlc_maximum_msat` (type `msat`, sqltype `INTEGER`, from JSON object `local`)\n - `local_cltv_expiry_delta` (type `u32`, sqltype `INTEGER`, from JSON object `local`)\n - `local_fee_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `local`)\n - `local_fee_proportional_millionths` (type `u32`, sqltype `INTEGER`, from JSON object `local`)\n - `remote_htlc_minimum_msat` (type `msat`, sqltype `INTEGER`, from JSON object `remote`)\n - `remote_htlc_maximum_msat` (type `msat`, sqltype `INTEGER`, from JSON object `remote`)\n - `remote_cltv_expiry_delta` (type `u32`, sqltype `INTEGER`, from JSON object `remote`)\n - `remote_fee_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `remote`)\n - `remote_fee_proportional_millionths` (type `u32`, sqltype `INTEGER`, from JSON object `remote`)\n - `ignore_fee_limits` (type `boolean`, sqltype `INTEGER`)\n - `lost_state` (type `boolean`, sqltype `INTEGER`)\n - `feerate_perkw` (type `u32`, sqltype `INTEGER`, from JSON object `feerate`)\n - `feerate_perkb` (type `u32`, sqltype `INTEGER`, from JSON object `feerate`)\n - `owner` (type `string`, sqltype `TEXT`)\n - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`)\n - `channel_id` (type `hash`, sqltype `BLOB`)\n - `funding_txid` (type `txid`, sqltype `BLOB`)\n - `funding_outnum` (type `u32`, sqltype `INTEGER`)\n - `initial_feerate` (type `string`, sqltype `TEXT`)\n - `last_feerate` (type `string`, sqltype `TEXT`)\n - `next_feerate` (type `string`, sqltype `TEXT`)\n - `next_fee_step` (type `u32`, sqltype `INTEGER`)\n - related table `peerchannels_inflight`\n - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `funding_txid` (type `txid`, sqltype `BLOB`)\n - `funding_outnum` (type `u32`, sqltype `INTEGER`)\n - `feerate` (type `string`, sqltype `TEXT`)\n - `total_funding_msat` (type `msat`, sqltype `INTEGER`)\n - `splice_amount` (type `integer`, sqltype `INTEGER`)\n - `our_funding_msat` (type `msat`, sqltype `INTEGER`)\n - `scratch_txid` (type `txid`, sqltype `BLOB`)\n - `close_to` (type `hex`, sqltype `BLOB`)\n - `private` (type `boolean`, sqltype `INTEGER`)\n - `opener` (type `string`, sqltype `TEXT`)\n - `closer` (type `string`, sqltype `TEXT`)\n - related table `peerchannels_features`\n - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `features` (type `string`, sqltype `TEXT`)\n - `funding_pushed_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`)\n - `funding_local_funds_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`)\n - `funding_remote_funds_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`)\n - `funding_fee_paid_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`)\n - `funding_fee_rcvd_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`)\n - `to_us_msat` (type `msat`, sqltype `INTEGER`)\n - `min_to_us_msat` (type `msat`, sqltype `INTEGER`)\n - `max_to_us_msat` (type `msat`, sqltype `INTEGER`)\n - `total_msat` (type `msat`, sqltype `INTEGER`)\n - `fee_base_msat` (type `msat`, sqltype `INTEGER`)\n - `fee_proportional_millionths` (type `u32`, sqltype `INTEGER`)\n - `dust_limit_msat` (type `msat`, sqltype `INTEGER`)\n - `max_total_htlc_in_msat` (type `msat`, sqltype `INTEGER`)\n - `their_reserve_msat` (type `msat`, sqltype `INTEGER`)\n - `our_reserve_msat` (type `msat`, sqltype `INTEGER`)\n - `spendable_msat` (type `msat`, sqltype `INTEGER`)\n - `receivable_msat` (type `msat`, sqltype `INTEGER`)\n - `minimum_htlc_in_msat` (type `msat`, sqltype `INTEGER`)\n - `minimum_htlc_out_msat` (type `msat`, sqltype `INTEGER`)\n - `maximum_htlc_out_msat` (type `msat`, sqltype `INTEGER`)\n - `their_to_self_delay` (type `u32`, sqltype `INTEGER`)\n - `our_to_self_delay` (type `u32`, sqltype `INTEGER`)\n - `max_accepted_htlcs` (type `u32`, sqltype `INTEGER`)\n - `alias_local` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`)\n - `alias_remote` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`)\n - related table `peerchannels_state_changes`\n - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `timestamp` (type `string`, sqltype `TEXT`)\n - `old_state` (type `string`, sqltype `TEXT`)\n - `new_state` (type `string`, sqltype `TEXT`)\n - `cause` (type `string`, sqltype `TEXT`)\n - `message` (type `string`, sqltype `TEXT`)\n - related table `peerchannels_status`\n - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `status` (type `string`, sqltype `TEXT`)\n - `in_payments_offered` (type `u64`, sqltype `INTEGER`)\n - `in_offered_msat` (type `msat`, sqltype `INTEGER`)\n - `in_payments_fulfilled` (type `u64`, sqltype `INTEGER`)\n - `in_fulfilled_msat` (type `msat`, sqltype `INTEGER`)\n - `out_payments_offered` (type `u64`, sqltype `INTEGER`)\n - `out_offered_msat` (type `msat`, sqltype `INTEGER`)\n - `out_payments_fulfilled` (type `u64`, sqltype `INTEGER`)\n - `out_fulfilled_msat` (type `msat`, sqltype `INTEGER`)\n - `last_stable_connection` (type `u64`, sqltype `INTEGER`)\n - related table `peerchannels_htlcs`\n - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `direction` (type `string`, sqltype `TEXT`)\n - `id` (type `u64`, sqltype `INTEGER`)\n - `amount_msat` (type `msat`, sqltype `INTEGER`)\n - `expiry` (type `u32`, sqltype `INTEGER`)\n - `payment_hash` (type `hash`, sqltype `BLOB`)\n - `local_trimmed` (type `boolean`, sqltype `INTEGER`)\n - `status` (type `string`, sqltype `TEXT`)\n - `state` (type `string`, sqltype `TEXT`)\n - `close_to_addr` (type `string`, sqltype `TEXT`)\n - `last_tx_fee_msat` (type `msat`, sqltype `INTEGER`)\n - `direction` (type `u32`, sqltype `INTEGER`)\n\n- `peers` indexed by `id` (see lightning-listpeers(7))\n - `id` (type `pubkey`, sqltype `BLOB`)\n - `connected` (type `boolean`, sqltype `INTEGER`)\n - `num_channels` (type `u32`, sqltype `INTEGER`)\n - related table `peers_netaddr`\n - `row` (reference to `peers.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `netaddr` (type `string`, sqltype `TEXT`)\n - `remote_addr` (type `string`, sqltype `TEXT`)\n - `features` (type `hex`, sqltype `BLOB`)\n\n- `sendpays` indexed by `payment_hash` (see lightning-listsendpays(7))\n - `created_index` (type `u64`, sqltype `INTEGER`)\n - `id` (type `u64`, sqltype `INTEGER`)\n - `groupid` (type `u64`, sqltype `INTEGER`)\n - `partid` (type `u64`, sqltype `INTEGER`)\n - `payment_hash` (type `hash`, sqltype `BLOB`)\n - `updated_index` (type `u64`, sqltype `INTEGER`)\n - `status` (type `string`, sqltype `TEXT`)\n - `amount_msat` (type `msat`, sqltype `INTEGER`)\n - `destination` (type `pubkey`, sqltype `BLOB`)\n - `created_at` (type `u64`, sqltype `INTEGER`)\n - `amount_sent_msat` (type `msat`, sqltype `INTEGER`)\n - `label` (type `string`, sqltype `TEXT`)\n - `bolt11` (type `string`, sqltype `TEXT`)\n - `description` (type `string`, sqltype `TEXT`)\n - `bolt12` (type `string`, sqltype `TEXT`)\n - `completed_at` (type `u64`, sqltype `INTEGER`)\n - `payment_preimage` (type `secret`, sqltype `BLOB`)\n - `erroronion` (type `hex`, sqltype `BLOB`)\n\n- `transactions` indexed by `hash` (see lightning-listtransactions(7))\n - `hash` (type `txid`, sqltype `BLOB`)\n - `rawtx` (type `hex`, sqltype `BLOB`)\n - `blockheight` (type `u32`, sqltype `INTEGER`)\n - `txindex` (type `u32`, sqltype `INTEGER`)\n - `locktime` (type `u32`, sqltype `INTEGER`)\n - `version` (type `u32`, sqltype `INTEGER`)\n - related table `transactions_inputs`\n - `row` (reference to `transactions.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `txid` (type `txid`, sqltype `BLOB`)\n - `idx` (type `u32`, sqltype `INTEGER`, from JSON field `index`)\n - `sequence` (type `u32`, sqltype `INTEGER`)\n - related table `transactions_outputs`\n - `row` (reference to `transactions.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `idx` (type `u32`, sqltype `INTEGER`, from JSON field `index`)\n - `amount_msat` (type `msat`, sqltype `INTEGER`)\n - `scriptPubKey` (type `hex`, sqltype `BLOB`)" + "" ], "errors": [ "On failure, an error is returned." diff --git a/contrib/msggen/msggen/utils/utils.py b/contrib/msggen/msggen/utils/utils.py index 484e51c7f..4cdadccc5 100644 --- a/contrib/msggen/msggen/utils/utils.py +++ b/contrib/msggen/msggen/utils/utils.py @@ -12,7 +12,8 @@ def combine_schemas(schema_dir: Path, dest: Path): files = sorted(list(schema_dir.iterdir())) for f in files: - if not f.name.endswith(".json"): + # Ignore lightning-sql.json because it will be auto generated by sql plugin and lightning-sql-template.json + if not f.name.endswith(".json") or f.name == "lightning-sql.json": continue bundle[f.name] = json.load(f.open()) diff --git a/doc/Makefile b/doc/Makefile index 29a89d8ae..a3f58c32a 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -141,12 +141,11 @@ MANPAGES := $(GENERATE_MARKDOWN) \ MARKDOWN_WITH_SCHEMA := $(GENERATE_MARKDOWN:=.md) -doc/schemas/lightning-sql.json: plugins/sql -# Delete the old tables schema and generate the new ones. - @if [ "$$(jq '[.tables[] | startswith("The following tables are currently supported:")]' doc/schemas/lightning-sql.json | jq 'any')" = "true" ]; then \ - jq 'del(.tables[] | select(startswith("The following tables are currently supported:")))' "$@" > "$@.tmp" && mv "$@.tmp" "$@"; \ - fi - @plugins/sql --print-docs | jq --arg sqldata "$$(awk '{printf "%s\n", $$0}')" '.tables += [$$sqldata]' "$@" > "$@.tmp" && mv "$@.tmp" "$@"; +# - Read the json template from schemas/lightning-sql-template.json +# - Generate the tables schema via plugins/sql +# - Merge both and generate final schemas/lightning-sql.json +doc/schemas/lightning-sql.json: doc/schemas/lightning-sql-template.json plugins/sql + @plugins/sql --print-docs | jq --arg sqldata "$$(awk '{printf "%s\n", $$0}')" '.tables += [$$sqldata]' $< > "$@.tmp" && mv "$@.tmp" "$@"; doc-all: $(MANPAGES) doc/index.rst diff --git a/doc/lightning-sql.7.md b/doc/lightning-sql.7.md index 1bb81e3de..4bb1d2f12 100644 --- a/doc/lightning-sql.7.md +++ b/doc/lightning-sql.7.md @@ -20,6 +20,388 @@ It is, however faster for remote access if the result of the query is much small - **query** (string): The standard sqlite3 query to run. Note that queries like "SELECT *" are fragile, as columns will change across releases; see lightning-listsqlschemas(7). +PERMITTED SQLITE3 FUNCTIONS +--------------------------- + +Writing to the database is not permitted, and limits are placed on various other query parameters. + +Additionally, only the following functions are allowed: + +* abs +* avg +* coalesce +* count +* hex +* quote +* length +* like +* lower +* upper +* min +* max +* sum +* total + +TREATMENT OF TYPES +------------------ + +The following types are supported in schemas, and this shows how they are presented in the database. This matters: a JSON boolean is represented as an integer in the database, so a query will return 0 or 1, not true or false. + +* *hex*. A hex string. + * JSON: a string + * sqlite3: BLOB + +* *hash*/*secret*/*pubkey*/*txid*: just like *hex*. + +* *msat*/*integer*/*u64*/*u32*/*u16*/*u8*. Normal numbers. + * JSON: an unsigned integer + * sqlite3: INTEGER + +* *boolean*. True or false. + * JSON: literal **true** or **false** + * sqlite3: INTEGER + +* *number*. A floating point number (used for times in some places). + * JSON: number + * sqlite3: REAL + +* *string*. Text. + * JSON: string + * sqlite3: TEXT + +* *short\_channel\_id*. A short-channel-id of form 1x2x3. + * JSON: string + * sqlite3: TEXT + +TABLES +------ + +Note that the first column of every table is a unique integer called `rowid`: this is used for related tables to refer to specific rows in their parent. sqlite3 usually has this as an implicit column, but we make it explicit as the implicit version is not allowed to be used as a foreign key. + +The following tables are currently supported: +- `bkpr_accountevents` (see lightning-bkpr-listaccountevents(7)) + - `account` (type `string`, sqltype `TEXT`) + - `type` (type `string`, sqltype `TEXT`) + - `tag` (type `string`, sqltype `TEXT`) + - `credit_msat` (type `msat`, sqltype `INTEGER`) + - `debit_msat` (type `msat`, sqltype `INTEGER`) + - `currency` (type `string`, sqltype `TEXT`) + - `timestamp` (type `u32`, sqltype `INTEGER`) + - `outpoint` (type `string`, sqltype `TEXT`) + - `blockheight` (type `u32`, sqltype `INTEGER`) + - `origin` (type `string`, sqltype `TEXT`) + - `payment_id` (type `hex`, sqltype `BLOB`) + - `txid` (type `txid`, sqltype `BLOB`) + - `description` (type `string`, sqltype `TEXT`) + - `fees_msat` (type `msat`, sqltype `INTEGER`) + - `is_rebalance` (type `boolean`, sqltype `INTEGER`) + - `part_id` (type `u32`, sqltype `INTEGER`) + +- `bkpr_income` (see lightning-bkpr-listincome(7)) + - `account` (type `string`, sqltype `TEXT`) + - `tag` (type `string`, sqltype `TEXT`) + - `credit_msat` (type `msat`, sqltype `INTEGER`) + - `debit_msat` (type `msat`, sqltype `INTEGER`) + - `currency` (type `string`, sqltype `TEXT`) + - `timestamp` (type `u32`, sqltype `INTEGER`) + - `description` (type `string`, sqltype `TEXT`) + - `outpoint` (type `string`, sqltype `TEXT`) + - `txid` (type `txid`, sqltype `BLOB`) + - `payment_id` (type `hex`, sqltype `BLOB`) + +- `channels` indexed by `short_channel_id` (see lightning-listchannels(7)) + - `source` (type `pubkey`, sqltype `BLOB`) + - `destination` (type `pubkey`, sqltype `BLOB`) + - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`) + - `direction` (type `u32`, sqltype `INTEGER`) + - `public` (type `boolean`, sqltype `INTEGER`) + - `amount_msat` (type `msat`, sqltype `INTEGER`) + - `message_flags` (type `u8`, sqltype `INTEGER`) + - `channel_flags` (type `u8`, sqltype `INTEGER`) + - `active` (type `boolean`, sqltype `INTEGER`) + - `last_update` (type `u32`, sqltype `INTEGER`) + - `base_fee_millisatoshi` (type `u32`, sqltype `INTEGER`) + - `fee_per_millionth` (type `u32`, sqltype `INTEGER`) + - `delay` (type `u32`, sqltype `INTEGER`) + - `htlc_minimum_msat` (type `msat`, sqltype `INTEGER`) + - `htlc_maximum_msat` (type `msat`, sqltype `INTEGER`) + - `features` (type `hex`, sqltype `BLOB`) + +- `closedchannels` (see lightning-listclosedchannels(7)) + - `peer_id` (type `pubkey`, sqltype `BLOB`) + - `channel_id` (type `hash`, sqltype `BLOB`) + - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`) + - `alias_local` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`) + - `alias_remote` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`) + - `opener` (type `string`, sqltype `TEXT`) + - `closer` (type `string`, sqltype `TEXT`) + - `private` (type `boolean`, sqltype `INTEGER`) + - related table `closedchannels_channel_type_bits`, from JSON object `channel_type` + - `row` (reference to `closedchannels_channel_type.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `bits` (type `u32`, sqltype `INTEGER`) + - related table `closedchannels_channel_type_names`, from JSON object `channel_type` + - `row` (reference to `closedchannels_channel_type.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `names` (type `string`, sqltype `TEXT`) + - `total_local_commitments` (type `u64`, sqltype `INTEGER`) + - `total_remote_commitments` (type `u64`, sqltype `INTEGER`) + - `total_htlcs_sent` (type `u64`, sqltype `INTEGER`) + - `funding_txid` (type `txid`, sqltype `BLOB`) + - `funding_outnum` (type `u32`, sqltype `INTEGER`) + - `leased` (type `boolean`, sqltype `INTEGER`) + - `funding_fee_paid_msat` (type `msat`, sqltype `INTEGER`) + - `funding_fee_rcvd_msat` (type `msat`, sqltype `INTEGER`) + - `funding_pushed_msat` (type `msat`, sqltype `INTEGER`) + - `total_msat` (type `msat`, sqltype `INTEGER`) + - `final_to_us_msat` (type `msat`, sqltype `INTEGER`) + - `min_to_us_msat` (type `msat`, sqltype `INTEGER`) + - `max_to_us_msat` (type `msat`, sqltype `INTEGER`) + - `last_commitment_txid` (type `hash`, sqltype `BLOB`) + - `last_commitment_fee_msat` (type `msat`, sqltype `INTEGER`) + - `close_cause` (type `string`, sqltype `TEXT`) + - `last_stable_connection` (type `u64`, sqltype `INTEGER`) + +- `forwards` indexed by `in_channel and in_htlc_id` (see lightning-listforwards(7)) + - `created_index` (type `u64`, sqltype `INTEGER`) + - `in_channel` (type `short_channel_id`, sqltype `TEXT`) + - `in_htlc_id` (type `u64`, sqltype `INTEGER`) + - `in_msat` (type `msat`, sqltype `INTEGER`) + - `status` (type `string`, sqltype `TEXT`) + - `received_time` (type `number`, sqltype `REAL`) + - `out_channel` (type `short_channel_id`, sqltype `TEXT`) + - `out_htlc_id` (type `u64`, sqltype `INTEGER`) + - `updated_index` (type `u64`, sqltype `INTEGER`) + - `style` (type `string`, sqltype `TEXT`) + - `fee_msat` (type `msat`, sqltype `INTEGER`) + - `out_msat` (type `msat`, sqltype `INTEGER`) + - `resolved_time` (type `number`, sqltype `REAL`) + - `failcode` (type `u32`, sqltype `INTEGER`) + - `failreason` (type `string`, sqltype `TEXT`) + +- `htlcs` indexed by `short_channel_id and id` (see lightning-listhtlcs(7)) + - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`) + - `id` (type `u64`, sqltype `INTEGER`) + - `expiry` (type `u32`, sqltype `INTEGER`) + - `amount_msat` (type `msat`, sqltype `INTEGER`) + - `direction` (type `string`, sqltype `TEXT`) + - `payment_hash` (type `hash`, sqltype `BLOB`) + - `state` (type `string`, sqltype `TEXT`) + +- `invoices` indexed by `payment_hash` (see lightning-listinvoices(7)) + - `label` (type `string`, sqltype `TEXT`) + - `description` (type `string`, sqltype `TEXT`) + - `payment_hash` (type `hash`, sqltype `BLOB`) + - `status` (type `string`, sqltype `TEXT`) + - `expires_at` (type `u64`, sqltype `INTEGER`) + - `amount_msat` (type `msat`, sqltype `INTEGER`) + - `bolt11` (type `string`, sqltype `TEXT`) + - `bolt12` (type `string`, sqltype `TEXT`) + - `local_offer_id` (type `hash`, sqltype `BLOB`) + - `invreq_payer_note` (type `string`, sqltype `TEXT`) + - `created_index` (type `u64`, sqltype `INTEGER`) + - `updated_index` (type `u64`, sqltype `INTEGER`) + - `pay_index` (type `u64`, sqltype `INTEGER`) + - `amount_received_msat` (type `msat`, sqltype `INTEGER`) + - `paid_at` (type `u64`, sqltype `INTEGER`) + - `paid_outpoint_txid` (type `txid`, sqltype `BLOB`, from JSON object `paid_outpoint`) + - `paid_outpoint_outnum` (type `u32`, sqltype `INTEGER`, from JSON object `paid_outpoint`) + - `payment_preimage` (type `secret`, sqltype `BLOB`) + +- `nodes` indexed by `nodeid` (see lightning-listnodes(7)) + - `nodeid` (type `pubkey`, sqltype `BLOB`) + - `last_timestamp` (type `u32`, sqltype `INTEGER`) + - `alias` (type `string`, sqltype `TEXT`) + - `color` (type `hex`, sqltype `BLOB`) + - `features` (type `hex`, sqltype `BLOB`) + - related table `nodes_addresses` + - `row` (reference to `nodes.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `type` (type `string`, sqltype `TEXT`) + - `port` (type `u16`, sqltype `INTEGER`) + - `address` (type `string`, sqltype `TEXT`) + - `option_will_fund_lease_fee_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `option_will_fund`) + - `option_will_fund_lease_fee_basis` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`) + - `option_will_fund_funding_weight` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`) + - `option_will_fund_channel_fee_max_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `option_will_fund`) + - `option_will_fund_channel_fee_max_proportional_thousandths` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`) + - `option_will_fund_compact_lease` (type `hex`, sqltype `BLOB`, from JSON object `option_will_fund`) + +- `offers` indexed by `offer_id` (see lightning-listoffers(7)) + - `offer_id` (type `hash`, sqltype `BLOB`) + - `active` (type `boolean`, sqltype `INTEGER`) + - `single_use` (type `boolean`, sqltype `INTEGER`) + - `bolt12` (type `string`, sqltype `TEXT`) + - `used` (type `boolean`, sqltype `INTEGER`) + - `label` (type `string`, sqltype `TEXT`) + +- `peerchannels` indexed by `peer_id` (see lightning-listpeerchannels(7)) + - `peer_id` (type `pubkey`, sqltype `BLOB`) + - `peer_connected` (type `boolean`, sqltype `INTEGER`) + - `reestablished` (type `boolean`, sqltype `INTEGER`) + - `state` (type `string`, sqltype `TEXT`) + - `scratch_txid` (type `txid`, sqltype `BLOB`) + - related table `peerchannels_channel_type_bits`, from JSON object `channel_type` + - `row` (reference to `peerchannels_channel_type.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `bits` (type `u32`, sqltype `INTEGER`) + - related table `peerchannels_channel_type_names`, from JSON object `channel_type` + - `row` (reference to `peerchannels_channel_type.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `names` (type `string`, sqltype `TEXT`) + - `local_htlc_minimum_msat` (type `msat`, sqltype `INTEGER`, from JSON object `local`) + - `local_htlc_maximum_msat` (type `msat`, sqltype `INTEGER`, from JSON object `local`) + - `local_cltv_expiry_delta` (type `u32`, sqltype `INTEGER`, from JSON object `local`) + - `local_fee_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `local`) + - `local_fee_proportional_millionths` (type `u32`, sqltype `INTEGER`, from JSON object `local`) + - `remote_htlc_minimum_msat` (type `msat`, sqltype `INTEGER`, from JSON object `remote`) + - `remote_htlc_maximum_msat` (type `msat`, sqltype `INTEGER`, from JSON object `remote`) + - `remote_cltv_expiry_delta` (type `u32`, sqltype `INTEGER`, from JSON object `remote`) + - `remote_fee_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `remote`) + - `remote_fee_proportional_millionths` (type `u32`, sqltype `INTEGER`, from JSON object `remote`) + - `ignore_fee_limits` (type `boolean`, sqltype `INTEGER`) + - `lost_state` (type `boolean`, sqltype `INTEGER`) + - `feerate_perkw` (type `u32`, sqltype `INTEGER`, from JSON object `feerate`) + - `feerate_perkb` (type `u32`, sqltype `INTEGER`, from JSON object `feerate`) + - `owner` (type `string`, sqltype `TEXT`) + - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`) + - `channel_id` (type `hash`, sqltype `BLOB`) + - `funding_txid` (type `txid`, sqltype `BLOB`) + - `funding_outnum` (type `u32`, sqltype `INTEGER`) + - `initial_feerate` (type `string`, sqltype `TEXT`) + - `last_feerate` (type `string`, sqltype `TEXT`) + - `next_feerate` (type `string`, sqltype `TEXT`) + - `next_fee_step` (type `u32`, sqltype `INTEGER`) + - related table `peerchannels_inflight` + - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `funding_txid` (type `txid`, sqltype `BLOB`) + - `funding_outnum` (type `u32`, sqltype `INTEGER`) + - `feerate` (type `string`, sqltype `TEXT`) + - `total_funding_msat` (type `msat`, sqltype `INTEGER`) + - `splice_amount` (type `integer`, sqltype `INTEGER`) + - `our_funding_msat` (type `msat`, sqltype `INTEGER`) + - `scratch_txid` (type `txid`, sqltype `BLOB`) + - `close_to` (type `hex`, sqltype `BLOB`) + - `private` (type `boolean`, sqltype `INTEGER`) + - `opener` (type `string`, sqltype `TEXT`) + - `closer` (type `string`, sqltype `TEXT`) + - related table `peerchannels_features` + - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `features` (type `string`, sqltype `TEXT`) + - `funding_pushed_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`) + - `funding_local_funds_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`) + - `funding_remote_funds_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`) + - `funding_fee_paid_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`) + - `funding_fee_rcvd_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`) + - `to_us_msat` (type `msat`, sqltype `INTEGER`) + - `min_to_us_msat` (type `msat`, sqltype `INTEGER`) + - `max_to_us_msat` (type `msat`, sqltype `INTEGER`) + - `total_msat` (type `msat`, sqltype `INTEGER`) + - `fee_base_msat` (type `msat`, sqltype `INTEGER`) + - `fee_proportional_millionths` (type `u32`, sqltype `INTEGER`) + - `dust_limit_msat` (type `msat`, sqltype `INTEGER`) + - `max_total_htlc_in_msat` (type `msat`, sqltype `INTEGER`) + - `their_reserve_msat` (type `msat`, sqltype `INTEGER`) + - `our_reserve_msat` (type `msat`, sqltype `INTEGER`) + - `spendable_msat` (type `msat`, sqltype `INTEGER`) + - `receivable_msat` (type `msat`, sqltype `INTEGER`) + - `minimum_htlc_in_msat` (type `msat`, sqltype `INTEGER`) + - `minimum_htlc_out_msat` (type `msat`, sqltype `INTEGER`) + - `maximum_htlc_out_msat` (type `msat`, sqltype `INTEGER`) + - `their_to_self_delay` (type `u32`, sqltype `INTEGER`) + - `our_to_self_delay` (type `u32`, sqltype `INTEGER`) + - `max_accepted_htlcs` (type `u32`, sqltype `INTEGER`) + - `alias_local` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`) + - `alias_remote` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`) + - related table `peerchannels_state_changes` + - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `timestamp` (type `string`, sqltype `TEXT`) + - `old_state` (type `string`, sqltype `TEXT`) + - `new_state` (type `string`, sqltype `TEXT`) + - `cause` (type `string`, sqltype `TEXT`) + - `message` (type `string`, sqltype `TEXT`) + - related table `peerchannels_status` + - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `status` (type `string`, sqltype `TEXT`) + - `in_payments_offered` (type `u64`, sqltype `INTEGER`) + - `in_offered_msat` (type `msat`, sqltype `INTEGER`) + - `in_payments_fulfilled` (type `u64`, sqltype `INTEGER`) + - `in_fulfilled_msat` (type `msat`, sqltype `INTEGER`) + - `out_payments_offered` (type `u64`, sqltype `INTEGER`) + - `out_offered_msat` (type `msat`, sqltype `INTEGER`) + - `out_payments_fulfilled` (type `u64`, sqltype `INTEGER`) + - `out_fulfilled_msat` (type `msat`, sqltype `INTEGER`) + - `last_stable_connection` (type `u64`, sqltype `INTEGER`) + - related table `peerchannels_htlcs` + - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `direction` (type `string`, sqltype `TEXT`) + - `id` (type `u64`, sqltype `INTEGER`) + - `amount_msat` (type `msat`, sqltype `INTEGER`) + - `expiry` (type `u32`, sqltype `INTEGER`) + - `payment_hash` (type `hash`, sqltype `BLOB`) + - `local_trimmed` (type `boolean`, sqltype `INTEGER`) + - `status` (type `string`, sqltype `TEXT`) + - `state` (type `string`, sqltype `TEXT`) + - `close_to_addr` (type `string`, sqltype `TEXT`) + - `last_tx_fee_msat` (type `msat`, sqltype `INTEGER`) + - `direction` (type `u32`, sqltype `INTEGER`) + +- `peers` indexed by `id` (see lightning-listpeers(7)) + - `id` (type `pubkey`, sqltype `BLOB`) + - `connected` (type `boolean`, sqltype `INTEGER`) + - `num_channels` (type `u32`, sqltype `INTEGER`) + - related table `peers_netaddr` + - `row` (reference to `peers.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `netaddr` (type `string`, sqltype `TEXT`) + - `remote_addr` (type `string`, sqltype `TEXT`) + - `features` (type `hex`, sqltype `BLOB`) + +- `sendpays` indexed by `payment_hash` (see lightning-listsendpays(7)) + - `created_index` (type `u64`, sqltype `INTEGER`) + - `id` (type `u64`, sqltype `INTEGER`) + - `groupid` (type `u64`, sqltype `INTEGER`) + - `partid` (type `u64`, sqltype `INTEGER`) + - `payment_hash` (type `hash`, sqltype `BLOB`) + - `updated_index` (type `u64`, sqltype `INTEGER`) + - `status` (type `string`, sqltype `TEXT`) + - `amount_msat` (type `msat`, sqltype `INTEGER`) + - `destination` (type `pubkey`, sqltype `BLOB`) + - `created_at` (type `u64`, sqltype `INTEGER`) + - `amount_sent_msat` (type `msat`, sqltype `INTEGER`) + - `label` (type `string`, sqltype `TEXT`) + - `bolt11` (type `string`, sqltype `TEXT`) + - `description` (type `string`, sqltype `TEXT`) + - `bolt12` (type `string`, sqltype `TEXT`) + - `completed_at` (type `u64`, sqltype `INTEGER`) + - `payment_preimage` (type `secret`, sqltype `BLOB`) + - `erroronion` (type `hex`, sqltype `BLOB`) + +- `transactions` indexed by `hash` (see lightning-listtransactions(7)) + - `hash` (type `txid`, sqltype `BLOB`) + - `rawtx` (type `hex`, sqltype `BLOB`) + - `blockheight` (type `u32`, sqltype `INTEGER`) + - `txindex` (type `u32`, sqltype `INTEGER`) + - `locktime` (type `u32`, sqltype `INTEGER`) + - `version` (type `u32`, sqltype `INTEGER`) + - related table `transactions_inputs` + - `row` (reference to `transactions.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `txid` (type `txid`, sqltype `BLOB`) + - `idx` (type `u32`, sqltype `INTEGER`, from JSON field `index`) + - `sequence` (type `u32`, sqltype `INTEGER`) + - related table `transactions_outputs` + - `row` (reference to `transactions.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `idx` (type `u32`, sqltype `INTEGER`, from JSON field `index`) + - `amount_msat` (type `msat`, sqltype `INTEGER`) + - `scriptPubKey` (type `hex`, sqltype `BLOB`) + EXAMPLE USAGE ------------- diff --git a/doc/schemas/lightning-sql-template.json b/doc/schemas/lightning-sql-template.json new file mode 100644 index 000000000..7b9cd43e5 --- /dev/null +++ b/doc/schemas/lightning-sql-template.json @@ -0,0 +1,265 @@ +{ + "$schema": "../rpc-schema-draft.json", + "type": "object", + "additionalProperties": false, + "added": "v23.02", + "rpc": "sql", + "title": "Command to do complex queries on list commands", + "description": [ + "The **sql** RPC command runs the given query across a sqlite3 database created from various list commands.", + "", + "When tables are accessed, it calls the below commands, so it's no faster than any other local access (though it goes to great length to cache `listnodes` and `listchannels`) which then processes the results.", + "", + "It is, however faster for remote access if the result of the query is much smaller than the list commands would be." + ], + "request": { + "required": [ + "query" + ], + "properties": { + "query": { + "type": "string", + "description": [ + "The standard sqlite3 query to run.", + "Note that queries like \"SELECT *\" are fragile, as columns will change across releases; see lightning-listsqlschemas(7)." + ] + } + } + }, + "response": { + "required": [ + "rows" + ], + "properties": { + "rows": { + "type": "array", + "items": { + "type": "array" + } + }, + "warning_db_failure": { + "type": "string", + "description": [ + "A message if the database encounters an error partway through." + ] + } + }, + "pre_return_value_notes": [ + "On success, an object containing **rows** is returned. It is an array. Each array entry contains an array of values, each an integer, real number, string or *null*, depending on the sqlite3 type.", + "", + "The object may contain **warning_db_failure** if the database fails partway through its operation." + ] + }, + "treatment_of_types": [ + "The following types are supported in schemas, and this shows how they are presented in the database. This matters: a JSON boolean is represented as an integer in the database, so a query will return 0 or 1, not true or false.", + "", + "* *hex*. A hex string.", + " * JSON: a string", + " * sqlite3: BLOB", + "", + "* *hash*/*secret*/*pubkey*/*txid*: just like *hex*.", + "", + "* *msat*/*integer*/*u64*/*u32*/*u16*/*u8*. Normal numbers.", + " * JSON: an unsigned integer", + " * sqlite3: INTEGER", + "", + "* *boolean*. True or false.", + " * JSON: literal **true** or **false**", + " * sqlite3: INTEGER", + "", + "* *number*. A floating point number (used for times in some places).", + " * JSON: number", + " * sqlite3: REAL", + "", + "* *string*. Text.", + " * JSON: string", + " * sqlite3: TEXT", + "", + "* *short_channel_id*. A short-channel-id of form 1x2x3.", + " * JSON: string", + " * sqlite3: TEXT" + ], + "permitted_sqlite3_functions": [ + "Writing to the database is not permitted, and limits are placed on various other query parameters.", + "", + "Additionally, only the following functions are allowed:", + "", + "* abs", + "* avg", + "* coalesce", + "* count", + "* hex", + "* quote", + "* length", + "* like", + "* lower", + "* upper", + "* min", + "* max", + "* sum", + "* total" + ], + "tables": [ + "Note that the first column of every table is a unique integer called `rowid`: this is used for related tables to refer to specific rows in their parent. sqlite3 usually has this as an implicit column, but we make it explicit as the implicit version is not allowed to be used as a foreign key.", + "" + ], + "errors": [ + "On failure, an error is returned." + ], + "example_usage": [ + "Here are some example using lightning-cli. Note that you may need to use `-o` if you use queries which contain `=` (which make lightning-cli(1) default to keyword style):", + "", + "A simple peer selection query:", + "", + "```shell", + "$ lightning-cli sql \"SELECT id FROM peers\"", + "{", + " \"rows\": [", + " [", + " \"02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00\"", + " ]", + " ]", + "}", + "```", + "", + "A statement containing using `=` needs `-o`:", + "", + "```shell", + "$ lightning-cli sql -o \"SELECT node_id,last_timestamp FROM nodes WHERE last_timestamp>=1669578892\"", + "{", + " \"rows\": [", + " [", + " \"02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00\",", + " 1669601603", + " ]", + " ]", + "}", + "```", + "", + "If you want to compare a BLOB column, `x'hex'` or `X'hex'` are needed:", + "", + "```shell", + "$ lightning-cli sql -o \"SELECT nodeid FROM nodes WHERE nodeid != x'03c9d25b6c0ce4bde5ad97d7ab83f00ae8bd3800a98ccbee36f3c3205315147de1';\"", + "{", + " \"rows\": [", + " [", + " \"0214739d625944f8fdc0da9d2ef44dbd7af58443685e494117b51410c5c3ff973a\"", + " ],", + " [", + " \"02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00\"", + " ]", + " ]", + "}", + "$ lightning-cli sql -o \"SELECT nodeid FROM nodes WHERE nodeid IN (x'03c9d25b6c0ce4bde5ad97d7ab83f00ae8bd3800a98ccbee36f3c3205315147de1', x'02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00')\"", + "{", + " \"rows\": [", + " [", + " \"02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00\"", + " ],", + " [", + " \"03c9d25b6c0ce4bde5ad97d7ab83f00ae8bd3800a98ccbee36f3c3205315147de1\"", + " ]", + " ]", + "}", + "```", + "", + "Related tables are usually referenced by JOIN:", + "", + "```shell", + "$ lightning-cli sql -o \"SELECT nodeid, alias, nodes_addresses.type, nodes_addresses.port, nodes_addresses.address FROM nodes INNER JOIN nodes_addresses ON nodes_addresses.row = nodes.rowid\"", + "{", + " \"rows\": [", + " [", + " \"02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00\",", + " \"YELLOWWATCH-22.11rc2-31-gcd7593b\",", + " \"dns\",", + " 7272,", + " \"localhost\"", + " ],", + " [", + " \"0214739d625944f8fdc0da9d2ef44dbd7af58443685e494117b51410c5c3ff973a\",", + " \"HOPPINGSQUIRREL-1rc2-31-gcd7593b\",", + " \"dns\",", + " 7171,", + " \"localhost\"", + " ]", + " ]", + "}", + "```", + "", + "Simple function usage, in this case COUNT. Strings inside arrays need \", and ' to protect them from the shell:", + "", + "```shell", + "$ lightning-cli sql 'SELECT COUNT(*) FROM nodes\"", + "{", + " \"rows\": [", + " [", + " 3", + " ]", + " ]", + "}", + "```" + ], + "example_json_request": [ + { + "id": "example:sql#1", + "method": "sql", + "params": [ + "SELECT * FROM forwards;" + ] + }, + { + "id": "example:sql#2", + "method": "sql", + "params": [ + "SELECT * from peerchannels_features" + ] + } + ], + "example_json_response": [ + { + "rows": [] + }, + { + "rows": [ + [ + 6, + 1, + 0, + "option_static_remotekey" + ], + [ + 7, + 1, + 1, + "option_anchors_zero_fee_htlc_tx" + ], + [ + 16, + 11, + 0, + "option_static_remotekey" + ], + [ + 17, + 11, + 1, + "option_anchors_zero_fee_htlc_tx" + ] + ] + } + ], + "author": [ + "Rusty Russell <> is mainly responsible." + ], + "see_also": [ + "lightning-listtransactions(7)", + "lightning-listchannels(7)", + "lightning-listpeers(7)", + "lightning-listnodes(7)", + "lightning-listforwards(7)" + ], + "resources": [ + "Main web site: " + ] +} diff --git a/doc/schemas/lightning-sql.json b/doc/schemas/lightning-sql.json deleted file mode 100644 index 63088cbbc..000000000 --- a/doc/schemas/lightning-sql.json +++ /dev/null @@ -1,266 +0,0 @@ -{ - "$schema": "../rpc-schema-draft.json", - "type": "object", - "additionalProperties": false, - "added": "v23.02", - "rpc": "sql", - "title": "Command to do complex queries on list commands", - "description": [ - "The **sql** RPC command runs the given query across a sqlite3 database created from various list commands.", - "", - "When tables are accessed, it calls the below commands, so it's no faster than any other local access (though it goes to great length to cache `listnodes` and `listchannels`) which then processes the results.", - "", - "It is, however faster for remote access if the result of the query is much smaller than the list commands would be." - ], - "request": { - "required": [ - "query" - ], - "properties": { - "query": { - "type": "string", - "description": [ - "The standard sqlite3 query to run.", - "Note that queries like \"SELECT *\" are fragile, as columns will change across releases; see lightning-listsqlschemas(7)." - ] - } - } - }, - "response": { - "required": [ - "rows" - ], - "properties": { - "rows": { - "type": "array", - "items": { - "type": "array" - } - }, - "warning_db_failure": { - "type": "string", - "description": [ - "A message if the database encounters an error partway through." - ] - } - }, - "pre_return_value_notes": [ - "On success, an object containing **rows** is returned. It is an array. Each array entry contains an array of values, each an integer, real number, string or *null*, depending on the sqlite3 type.", - "", - "The object may contain **warning_db_failure** if the database fails partway through its operation." - ] - }, - "treatment_of_types": [ - "The following types are supported in schemas, and this shows how they are presented in the database. This matters: a JSON boolean is represented as an integer in the database, so a query will return 0 or 1, not true or false.", - "", - "* *hex*. A hex string.", - " * JSON: a string", - " * sqlite3: BLOB", - "", - "* *hash*/*secret*/*pubkey*/*txid*: just like *hex*.", - "", - "* *msat*/*integer*/*u64*/*u32*/*u16*/*u8*. Normal numbers.", - " * JSON: an unsigned integer", - " * sqlite3: INTEGER", - "", - "* *boolean*. True or false.", - " * JSON: literal **true** or **false**", - " * sqlite3: INTEGER", - "", - "* *number*. A floating point number (used for times in some places).", - " * JSON: number", - " * sqlite3: REAL", - "", - "* *string*. Text.", - " * JSON: string", - " * sqlite3: TEXT", - "", - "* *short_channel_id*. A short-channel-id of form 1x2x3.", - " * JSON: string", - " * sqlite3: TEXT" - ], - "permitted_sqlite3_functions": [ - "Writing to the database is not permitted, and limits are placed on various other query parameters.", - "", - "Additionally, only the following functions are allowed:", - "", - "* abs", - "* avg", - "* coalesce", - "* count", - "* hex", - "* quote", - "* length", - "* like", - "* lower", - "* upper", - "* min", - "* max", - "* sum", - "* total" - ], - "tables": [ - "Note that the first column of every table is a unique integer called `rowid`: this is used for related tables to refer to specific rows in their parent. sqlite3 usually has this as an implicit column, but we make it explicit as the implicit version is not allowed to be used as a foreign key.", - "", - "The following tables are currently supported:\n- `bkpr_accountevents` (see lightning-bkpr-listaccountevents(7))\n - `account` (type `string`, sqltype `TEXT`)\n - `type` (type `string`, sqltype `TEXT`)\n - `tag` (type `string`, sqltype `TEXT`)\n - `credit_msat` (type `msat`, sqltype `INTEGER`)\n - `debit_msat` (type `msat`, sqltype `INTEGER`)\n - `currency` (type `string`, sqltype `TEXT`)\n - `timestamp` (type `u32`, sqltype `INTEGER`)\n - `outpoint` (type `string`, sqltype `TEXT`)\n - `blockheight` (type `u32`, sqltype `INTEGER`)\n - `origin` (type `string`, sqltype `TEXT`)\n - `payment_id` (type `hex`, sqltype `BLOB`)\n - `txid` (type `txid`, sqltype `BLOB`)\n - `description` (type `string`, sqltype `TEXT`)\n - `fees_msat` (type `msat`, sqltype `INTEGER`)\n - `is_rebalance` (type `boolean`, sqltype `INTEGER`)\n - `part_id` (type `u32`, sqltype `INTEGER`)\n\n- `bkpr_income` (see lightning-bkpr-listincome(7))\n - `account` (type `string`, sqltype `TEXT`)\n - `tag` (type `string`, sqltype `TEXT`)\n - `credit_msat` (type `msat`, sqltype `INTEGER`)\n - `debit_msat` (type `msat`, sqltype `INTEGER`)\n - `currency` (type `string`, sqltype `TEXT`)\n - `timestamp` (type `u32`, sqltype `INTEGER`)\n - `description` (type `string`, sqltype `TEXT`)\n - `outpoint` (type `string`, sqltype `TEXT`)\n - `txid` (type `txid`, sqltype `BLOB`)\n - `payment_id` (type `hex`, sqltype `BLOB`)\n\n- `channels` indexed by `short_channel_id` (see lightning-listchannels(7))\n - `source` (type `pubkey`, sqltype `BLOB`)\n - `destination` (type `pubkey`, sqltype `BLOB`)\n - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`)\n - `direction` (type `u32`, sqltype `INTEGER`)\n - `public` (type `boolean`, sqltype `INTEGER`)\n - `amount_msat` (type `msat`, sqltype `INTEGER`)\n - `message_flags` (type `u8`, sqltype `INTEGER`)\n - `channel_flags` (type `u8`, sqltype `INTEGER`)\n - `active` (type `boolean`, sqltype `INTEGER`)\n - `last_update` (type `u32`, sqltype `INTEGER`)\n - `base_fee_millisatoshi` (type `u32`, sqltype `INTEGER`)\n - `fee_per_millionth` (type `u32`, sqltype `INTEGER`)\n - `delay` (type `u32`, sqltype `INTEGER`)\n - `htlc_minimum_msat` (type `msat`, sqltype `INTEGER`)\n - `htlc_maximum_msat` (type `msat`, sqltype `INTEGER`)\n - `features` (type `hex`, sqltype `BLOB`)\n\n- `closedchannels` (see lightning-listclosedchannels(7))\n - `peer_id` (type `pubkey`, sqltype `BLOB`)\n - `channel_id` (type `hash`, sqltype `BLOB`)\n - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`)\n - `alias_local` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`)\n - `alias_remote` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`)\n - `opener` (type `string`, sqltype `TEXT`)\n - `closer` (type `string`, sqltype `TEXT`)\n - `private` (type `boolean`, sqltype `INTEGER`)\n - related table `closedchannels_channel_type_bits`, from JSON object `channel_type`\n - `row` (reference to `closedchannels_channel_type.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `bits` (type `u32`, sqltype `INTEGER`)\n - related table `closedchannels_channel_type_names`, from JSON object `channel_type`\n - `row` (reference to `closedchannels_channel_type.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `names` (type `string`, sqltype `TEXT`)\n - `total_local_commitments` (type `u64`, sqltype `INTEGER`)\n - `total_remote_commitments` (type `u64`, sqltype `INTEGER`)\n - `total_htlcs_sent` (type `u64`, sqltype `INTEGER`)\n - `funding_txid` (type `txid`, sqltype `BLOB`)\n - `funding_outnum` (type `u32`, sqltype `INTEGER`)\n - `leased` (type `boolean`, sqltype `INTEGER`)\n - `funding_fee_paid_msat` (type `msat`, sqltype `INTEGER`)\n - `funding_fee_rcvd_msat` (type `msat`, sqltype `INTEGER`)\n - `funding_pushed_msat` (type `msat`, sqltype `INTEGER`)\n - `total_msat` (type `msat`, sqltype `INTEGER`)\n - `final_to_us_msat` (type `msat`, sqltype `INTEGER`)\n - `min_to_us_msat` (type `msat`, sqltype `INTEGER`)\n - `max_to_us_msat` (type `msat`, sqltype `INTEGER`)\n - `last_commitment_txid` (type `hash`, sqltype `BLOB`)\n - `last_commitment_fee_msat` (type `msat`, sqltype `INTEGER`)\n - `close_cause` (type `string`, sqltype `TEXT`)\n - `last_stable_connection` (type `u64`, sqltype `INTEGER`)\n\n- `forwards` indexed by `in_channel and in_htlc_id` (see lightning-listforwards(7))\n - `created_index` (type `u64`, sqltype `INTEGER`)\n - `in_channel` (type `short_channel_id`, sqltype `TEXT`)\n - `in_htlc_id` (type `u64`, sqltype `INTEGER`)\n - `in_msat` (type `msat`, sqltype `INTEGER`)\n - `status` (type `string`, sqltype `TEXT`)\n - `received_time` (type `number`, sqltype `REAL`)\n - `out_channel` (type `short_channel_id`, sqltype `TEXT`)\n - `out_htlc_id` (type `u64`, sqltype `INTEGER`)\n - `updated_index` (type `u64`, sqltype `INTEGER`)\n - `style` (type `string`, sqltype `TEXT`)\n - `fee_msat` (type `msat`, sqltype `INTEGER`)\n - `out_msat` (type `msat`, sqltype `INTEGER`)\n - `resolved_time` (type `number`, sqltype `REAL`)\n - `failcode` (type `u32`, sqltype `INTEGER`)\n - `failreason` (type `string`, sqltype `TEXT`)\n\n- `htlcs` indexed by `short_channel_id and id` (see lightning-listhtlcs(7))\n - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`)\n - `id` (type `u64`, sqltype `INTEGER`)\n - `expiry` (type `u32`, sqltype `INTEGER`)\n - `amount_msat` (type `msat`, sqltype `INTEGER`)\n - `direction` (type `string`, sqltype `TEXT`)\n - `payment_hash` (type `hash`, sqltype `BLOB`)\n - `state` (type `string`, sqltype `TEXT`)\n\n- `invoices` indexed by `payment_hash` (see lightning-listinvoices(7))\n - `label` (type `string`, sqltype `TEXT`)\n - `description` (type `string`, sqltype `TEXT`)\n - `payment_hash` (type `hash`, sqltype `BLOB`)\n - `status` (type `string`, sqltype `TEXT`)\n - `expires_at` (type `u64`, sqltype `INTEGER`)\n - `amount_msat` (type `msat`, sqltype `INTEGER`)\n - `bolt11` (type `string`, sqltype `TEXT`)\n - `bolt12` (type `string`, sqltype `TEXT`)\n - `local_offer_id` (type `hash`, sqltype `BLOB`)\n - `invreq_payer_note` (type `string`, sqltype `TEXT`)\n - `created_index` (type `u64`, sqltype `INTEGER`)\n - `updated_index` (type `u64`, sqltype `INTEGER`)\n - `pay_index` (type `u64`, sqltype `INTEGER`)\n - `amount_received_msat` (type `msat`, sqltype `INTEGER`)\n - `paid_at` (type `u64`, sqltype `INTEGER`)\n - `paid_outpoint_txid` (type `txid`, sqltype `BLOB`, from JSON object `paid_outpoint`)\n - `paid_outpoint_outnum` (type `u32`, sqltype `INTEGER`, from JSON object `paid_outpoint`)\n - `payment_preimage` (type `secret`, sqltype `BLOB`)\n\n- `nodes` indexed by `nodeid` (see lightning-listnodes(7))\n - `nodeid` (type `pubkey`, sqltype `BLOB`)\n - `last_timestamp` (type `u32`, sqltype `INTEGER`)\n - `alias` (type `string`, sqltype `TEXT`)\n - `color` (type `hex`, sqltype `BLOB`)\n - `features` (type `hex`, sqltype `BLOB`)\n - related table `nodes_addresses`\n - `row` (reference to `nodes.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `type` (type `string`, sqltype `TEXT`)\n - `port` (type `u16`, sqltype `INTEGER`)\n - `address` (type `string`, sqltype `TEXT`)\n - `option_will_fund_lease_fee_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `option_will_fund`)\n - `option_will_fund_lease_fee_basis` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`)\n - `option_will_fund_funding_weight` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`)\n - `option_will_fund_channel_fee_max_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `option_will_fund`)\n - `option_will_fund_channel_fee_max_proportional_thousandths` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`)\n - `option_will_fund_compact_lease` (type `hex`, sqltype `BLOB`, from JSON object `option_will_fund`)\n\n- `offers` indexed by `offer_id` (see lightning-listoffers(7))\n - `offer_id` (type `hash`, sqltype `BLOB`)\n - `active` (type `boolean`, sqltype `INTEGER`)\n - `single_use` (type `boolean`, sqltype `INTEGER`)\n - `bolt12` (type `string`, sqltype `TEXT`)\n - `used` (type `boolean`, sqltype `INTEGER`)\n - `label` (type `string`, sqltype `TEXT`)\n\n- `peerchannels` indexed by `peer_id` (see lightning-listpeerchannels(7))\n - `peer_id` (type `pubkey`, sqltype `BLOB`)\n - `peer_connected` (type `boolean`, sqltype `INTEGER`)\n - `reestablished` (type `boolean`, sqltype `INTEGER`)\n - `state` (type `string`, sqltype `TEXT`)\n - `scratch_txid` (type `txid`, sqltype `BLOB`)\n - related table `peerchannels_channel_type_bits`, from JSON object `channel_type`\n - `row` (reference to `peerchannels_channel_type.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `bits` (type `u32`, sqltype `INTEGER`)\n - related table `peerchannels_channel_type_names`, from JSON object `channel_type`\n - `row` (reference to `peerchannels_channel_type.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `names` (type `string`, sqltype `TEXT`)\n - `local_htlc_minimum_msat` (type `msat`, sqltype `INTEGER`, from JSON object `local`)\n - `local_htlc_maximum_msat` (type `msat`, sqltype `INTEGER`, from JSON object `local`)\n - `local_cltv_expiry_delta` (type `u32`, sqltype `INTEGER`, from JSON object `local`)\n - `local_fee_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `local`)\n - `local_fee_proportional_millionths` (type `u32`, sqltype `INTEGER`, from JSON object `local`)\n - `remote_htlc_minimum_msat` (type `msat`, sqltype `INTEGER`, from JSON object `remote`)\n - `remote_htlc_maximum_msat` (type `msat`, sqltype `INTEGER`, from JSON object `remote`)\n - `remote_cltv_expiry_delta` (type `u32`, sqltype `INTEGER`, from JSON object `remote`)\n - `remote_fee_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `remote`)\n - `remote_fee_proportional_millionths` (type `u32`, sqltype `INTEGER`, from JSON object `remote`)\n - `ignore_fee_limits` (type `boolean`, sqltype `INTEGER`)\n - `lost_state` (type `boolean`, sqltype `INTEGER`)\n - `feerate_perkw` (type `u32`, sqltype `INTEGER`, from JSON object `feerate`)\n - `feerate_perkb` (type `u32`, sqltype `INTEGER`, from JSON object `feerate`)\n - `owner` (type `string`, sqltype `TEXT`)\n - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`)\n - `channel_id` (type `hash`, sqltype `BLOB`)\n - `funding_txid` (type `txid`, sqltype `BLOB`)\n - `funding_outnum` (type `u32`, sqltype `INTEGER`)\n - `initial_feerate` (type `string`, sqltype `TEXT`)\n - `last_feerate` (type `string`, sqltype `TEXT`)\n - `next_feerate` (type `string`, sqltype `TEXT`)\n - `next_fee_step` (type `u32`, sqltype `INTEGER`)\n - related table `peerchannels_inflight`\n - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `funding_txid` (type `txid`, sqltype `BLOB`)\n - `funding_outnum` (type `u32`, sqltype `INTEGER`)\n - `feerate` (type `string`, sqltype `TEXT`)\n - `total_funding_msat` (type `msat`, sqltype `INTEGER`)\n - `splice_amount` (type `integer`, sqltype `INTEGER`)\n - `our_funding_msat` (type `msat`, sqltype `INTEGER`)\n - `scratch_txid` (type `txid`, sqltype `BLOB`)\n - `close_to` (type `hex`, sqltype `BLOB`)\n - `private` (type `boolean`, sqltype `INTEGER`)\n - `opener` (type `string`, sqltype `TEXT`)\n - `closer` (type `string`, sqltype `TEXT`)\n - related table `peerchannels_features`\n - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `features` (type `string`, sqltype `TEXT`)\n - `funding_pushed_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`)\n - `funding_local_funds_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`)\n - `funding_remote_funds_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`)\n - `funding_fee_paid_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`)\n - `funding_fee_rcvd_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`)\n - `to_us_msat` (type `msat`, sqltype `INTEGER`)\n - `min_to_us_msat` (type `msat`, sqltype `INTEGER`)\n - `max_to_us_msat` (type `msat`, sqltype `INTEGER`)\n - `total_msat` (type `msat`, sqltype `INTEGER`)\n - `fee_base_msat` (type `msat`, sqltype `INTEGER`)\n - `fee_proportional_millionths` (type `u32`, sqltype `INTEGER`)\n - `dust_limit_msat` (type `msat`, sqltype `INTEGER`)\n - `max_total_htlc_in_msat` (type `msat`, sqltype `INTEGER`)\n - `their_reserve_msat` (type `msat`, sqltype `INTEGER`)\n - `our_reserve_msat` (type `msat`, sqltype `INTEGER`)\n - `spendable_msat` (type `msat`, sqltype `INTEGER`)\n - `receivable_msat` (type `msat`, sqltype `INTEGER`)\n - `minimum_htlc_in_msat` (type `msat`, sqltype `INTEGER`)\n - `minimum_htlc_out_msat` (type `msat`, sqltype `INTEGER`)\n - `maximum_htlc_out_msat` (type `msat`, sqltype `INTEGER`)\n - `their_to_self_delay` (type `u32`, sqltype `INTEGER`)\n - `our_to_self_delay` (type `u32`, sqltype `INTEGER`)\n - `max_accepted_htlcs` (type `u32`, sqltype `INTEGER`)\n - `alias_local` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`)\n - `alias_remote` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`)\n - related table `peerchannels_state_changes`\n - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `timestamp` (type `string`, sqltype `TEXT`)\n - `old_state` (type `string`, sqltype `TEXT`)\n - `new_state` (type `string`, sqltype `TEXT`)\n - `cause` (type `string`, sqltype `TEXT`)\n - `message` (type `string`, sqltype `TEXT`)\n - related table `peerchannels_status`\n - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `status` (type `string`, sqltype `TEXT`)\n - `in_payments_offered` (type `u64`, sqltype `INTEGER`)\n - `in_offered_msat` (type `msat`, sqltype `INTEGER`)\n - `in_payments_fulfilled` (type `u64`, sqltype `INTEGER`)\n - `in_fulfilled_msat` (type `msat`, sqltype `INTEGER`)\n - `out_payments_offered` (type `u64`, sqltype `INTEGER`)\n - `out_offered_msat` (type `msat`, sqltype `INTEGER`)\n - `out_payments_fulfilled` (type `u64`, sqltype `INTEGER`)\n - `out_fulfilled_msat` (type `msat`, sqltype `INTEGER`)\n - `last_stable_connection` (type `u64`, sqltype `INTEGER`)\n - related table `peerchannels_htlcs`\n - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `direction` (type `string`, sqltype `TEXT`)\n - `id` (type `u64`, sqltype `INTEGER`)\n - `amount_msat` (type `msat`, sqltype `INTEGER`)\n - `expiry` (type `u32`, sqltype `INTEGER`)\n - `payment_hash` (type `hash`, sqltype `BLOB`)\n - `local_trimmed` (type `boolean`, sqltype `INTEGER`)\n - `status` (type `string`, sqltype `TEXT`)\n - `state` (type `string`, sqltype `TEXT`)\n - `close_to_addr` (type `string`, sqltype `TEXT`)\n - `last_tx_fee_msat` (type `msat`, sqltype `INTEGER`)\n - `direction` (type `u32`, sqltype `INTEGER`)\n\n- `peers` indexed by `id` (see lightning-listpeers(7))\n - `id` (type `pubkey`, sqltype `BLOB`)\n - `connected` (type `boolean`, sqltype `INTEGER`)\n - `num_channels` (type `u32`, sqltype `INTEGER`)\n - related table `peers_netaddr`\n - `row` (reference to `peers.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `netaddr` (type `string`, sqltype `TEXT`)\n - `remote_addr` (type `string`, sqltype `TEXT`)\n - `features` (type `hex`, sqltype `BLOB`)\n\n- `sendpays` indexed by `payment_hash` (see lightning-listsendpays(7))\n - `created_index` (type `u64`, sqltype `INTEGER`)\n - `id` (type `u64`, sqltype `INTEGER`)\n - `groupid` (type `u64`, sqltype `INTEGER`)\n - `partid` (type `u64`, sqltype `INTEGER`)\n - `payment_hash` (type `hash`, sqltype `BLOB`)\n - `updated_index` (type `u64`, sqltype `INTEGER`)\n - `status` (type `string`, sqltype `TEXT`)\n - `amount_msat` (type `msat`, sqltype `INTEGER`)\n - `destination` (type `pubkey`, sqltype `BLOB`)\n - `created_at` (type `u64`, sqltype `INTEGER`)\n - `amount_sent_msat` (type `msat`, sqltype `INTEGER`)\n - `label` (type `string`, sqltype `TEXT`)\n - `bolt11` (type `string`, sqltype `TEXT`)\n - `description` (type `string`, sqltype `TEXT`)\n - `bolt12` (type `string`, sqltype `TEXT`)\n - `completed_at` (type `u64`, sqltype `INTEGER`)\n - `payment_preimage` (type `secret`, sqltype `BLOB`)\n - `erroronion` (type `hex`, sqltype `BLOB`)\n\n- `transactions` indexed by `hash` (see lightning-listtransactions(7))\n - `hash` (type `txid`, sqltype `BLOB`)\n - `rawtx` (type `hex`, sqltype `BLOB`)\n - `blockheight` (type `u32`, sqltype `INTEGER`)\n - `txindex` (type `u32`, sqltype `INTEGER`)\n - `locktime` (type `u32`, sqltype `INTEGER`)\n - `version` (type `u32`, sqltype `INTEGER`)\n - related table `transactions_inputs`\n - `row` (reference to `transactions.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `txid` (type `txid`, sqltype `BLOB`)\n - `idx` (type `u32`, sqltype `INTEGER`, from JSON field `index`)\n - `sequence` (type `u32`, sqltype `INTEGER`)\n - related table `transactions_outputs`\n - `row` (reference to `transactions.rowid`, sqltype `INTEGER`)\n - `arrindex` (index within array, sqltype `INTEGER`)\n - `idx` (type `u32`, sqltype `INTEGER`, from JSON field `index`)\n - `amount_msat` (type `msat`, sqltype `INTEGER`)\n - `scriptPubKey` (type `hex`, sqltype `BLOB`)" - ], - "errors": [ - "On failure, an error is returned." - ], - "example_usage": [ - "Here are some example using lightning-cli. Note that you may need to use `-o` if you use queries which contain `=` (which make lightning-cli(1) default to keyword style):", - "", - "A simple peer selection query:", - "", - "```shell", - "$ lightning-cli sql \"SELECT id FROM peers\"", - "{", - " \"rows\": [", - " [", - " \"02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00\"", - " ]", - " ]", - "}", - "```", - "", - "A statement containing using `=` needs `-o`:", - "", - "```shell", - "$ lightning-cli sql -o \"SELECT node_id,last_timestamp FROM nodes WHERE last_timestamp>=1669578892\"", - "{", - " \"rows\": [", - " [", - " \"02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00\",", - " 1669601603", - " ]", - " ]", - "}", - "```", - "", - "If you want to compare a BLOB column, `x'hex'` or `X'hex'` are needed:", - "", - "```shell", - "$ lightning-cli sql -o \"SELECT nodeid FROM nodes WHERE nodeid != x'03c9d25b6c0ce4bde5ad97d7ab83f00ae8bd3800a98ccbee36f3c3205315147de1';\"", - "{", - " \"rows\": [", - " [", - " \"0214739d625944f8fdc0da9d2ef44dbd7af58443685e494117b51410c5c3ff973a\"", - " ],", - " [", - " \"02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00\"", - " ]", - " ]", - "}", - "$ lightning-cli sql -o \"SELECT nodeid FROM nodes WHERE nodeid IN (x'03c9d25b6c0ce4bde5ad97d7ab83f00ae8bd3800a98ccbee36f3c3205315147de1', x'02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00')\"", - "{", - " \"rows\": [", - " [", - " \"02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00\"", - " ],", - " [", - " \"03c9d25b6c0ce4bde5ad97d7ab83f00ae8bd3800a98ccbee36f3c3205315147de1\"", - " ]", - " ]", - "}", - "```", - "", - "Related tables are usually referenced by JOIN:", - "", - "```shell", - "$ lightning-cli sql -o \"SELECT nodeid, alias, nodes_addresses.type, nodes_addresses.port, nodes_addresses.address FROM nodes INNER JOIN nodes_addresses ON nodes_addresses.row = nodes.rowid\"", - "{", - " \"rows\": [", - " [", - " \"02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00\",", - " \"YELLOWWATCH-22.11rc2-31-gcd7593b\",", - " \"dns\",", - " 7272,", - " \"localhost\"", - " ],", - " [", - " \"0214739d625944f8fdc0da9d2ef44dbd7af58443685e494117b51410c5c3ff973a\",", - " \"HOPPINGSQUIRREL-1rc2-31-gcd7593b\",", - " \"dns\",", - " 7171,", - " \"localhost\"", - " ]", - " ]", - "}", - "```", - "", - "Simple function usage, in this case COUNT. Strings inside arrays need \", and ' to protect them from the shell:", - "", - "```shell", - "$ lightning-cli sql 'SELECT COUNT(*) FROM nodes\"", - "{", - " \"rows\": [", - " [", - " 3", - " ]", - " ]", - "}", - "```" - ], - "example_json_request": [ - { - "id": "example:sql#1", - "method": "sql", - "params": [ - "SELECT * FROM forwards;" - ] - }, - { - "id": "example:sql#2", - "method": "sql", - "params": [ - "SELECT * from peerchannels_features" - ] - } - ], - "example_json_response": [ - { - "rows": [] - }, - { - "rows": [ - [ - 6, - 1, - 0, - "option_static_remotekey" - ], - [ - 7, - 1, - 1, - "option_anchors_zero_fee_htlc_tx" - ], - [ - 16, - 11, - 0, - "option_static_remotekey" - ], - [ - 17, - 11, - 1, - "option_anchors_zero_fee_htlc_tx" - ] - ] - } - ], - "author": [ - "Rusty Russell <> is mainly responsible." - ], - "see_also": [ - "lightning-listtransactions(7)", - "lightning-listchannels(7)", - "lightning-listpeers(7)", - "lightning-listnodes(7)", - "lightning-listforwards(7)" - ], - "resources": [ - "Main web site: " - ] -} diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 7082baea6..ec82cfcc2 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3714,6 +3714,8 @@ def test_sql(node_factory, bitcoind): 'type': 'string'}, {'name': 'bolt12', 'type': 'string'}, + {'name': 'completed_at', + 'type': 'u64'}, {'name': 'payment_preimage', 'type': 'secret'}, {'name': 'erroronion', @@ -3724,6 +3726,8 @@ def test_sql(node_factory, bitcoind): 'type': 'pubkey'}, {'name': 'peer_connected', 'type': 'boolean'}, + {'name': 'reestablished', + 'type': 'boolean'}, {'name': 'state', 'type': 'string'}, {'name': 'scratch_txid', @@ -3850,8 +3854,6 @@ def test_sql(node_factory, bitcoind): 'type': 'msat'}, {'name': 'last_stable_connection', 'type': 'u64'}, - {'name': 'reestablished', - 'type': 'boolean'}, {'name': 'close_to_addr', 'type': 'string'}, {'name': 'last_tx_fee_msat', diff --git a/tools/fromschema.py b/tools/fromschema.py index 90c5c3505..f2c437641 100755 --- a/tools/fromschema.py +++ b/tools/fromschema.py @@ -7,7 +7,7 @@ import json import re # To maintain the sequence of the before return value (body) and after return value (footer) sections in the markdown file -BODY_KEY_SEQUENCE = ['reliability', 'usage', 'restriction_format', 'example_usage', 'example_json_request', 'notes', 'notifications', 'sharing_runes', 'riskfactor_effect_on_routing', 'recommended_riskfactor_values', 'optimality', 'randomization'] +BODY_KEY_SEQUENCE = ['reliability', 'usage', 'restriction_format', 'permitted_sqlite3_functions', 'treatment_of_types', 'tables', 'example_usage', 'example_json_request', 'notes', 'notifications', 'sharing_runes', 'riskfactor_effect_on_routing', 'recommended_riskfactor_values', 'optimality', 'randomization'] FOOTER_KEY_SEQUENCE = ['example_json_response', 'errors', 'example_json_notifications', 'trivia', 'author', 'see_also', 'resources']