From 0240c2493622a9d757d47fb24fac9cc48126816a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:18 +1030 Subject: [PATCH] plugins/sql: listsqlschemas command to retrieve schemas. Good for detection of what fields are present. Signed-off-by: Rusty Russell --- doc/Makefile | 1 + doc/index.rst | 1 + doc/lightning-listsqlschemas.7.md | 109 +++++++++++++++++++++++ doc/lightning-sql.7.md | 3 + doc/schemas/listsqlschemas.request.json | 10 +++ doc/schemas/listsqlschemas.schema.json | 67 ++++++++++++++ plugins/sql.c | 112 ++++++++++++++++++++++++ tests/test_plugin.py | 28 ++++++ 8 files changed, 331 insertions(+) create mode 100644 doc/lightning-listsqlschemas.7.md create mode 100644 doc/schemas/listsqlschemas.request.json create mode 100644 doc/schemas/listsqlschemas.schema.json diff --git a/doc/Makefile b/doc/Makefile index 174a94131..764e5c890 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -60,6 +60,7 @@ MANPAGES := doc/lightning-cli.1 \ doc/lightning-listpeers.7 \ doc/lightning-listpeerchannels.7 \ doc/lightning-listsendpays.7 \ + doc/lightning-listsqlschemas.7 \ doc/lightning-makesecret.7 \ doc/lightning-multifundchannel.7 \ doc/lightning-multiwithdraw.7 \ diff --git a/doc/index.rst b/doc/index.rst index 822679c10..2f05acd7f 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -88,6 +88,7 @@ Core Lightning Documentation lightning-listpeerchannels lightning-listpeers lightning-listsendpays + lightning-listsqlschemas lightning-listtransactions lightning-makesecret lightning-multifundchannel diff --git a/doc/lightning-listsqlschemas.7.md b/doc/lightning-listsqlschemas.7.md new file mode 100644 index 000000000..13e725fce --- /dev/null +++ b/doc/lightning-listsqlschemas.7.md @@ -0,0 +1,109 @@ +lightning-listsqlschemas -- Command to example lightning-sql schemas +==================================================================== + +SYNOPSIS +-------- + +**listsqlschemas** [*table*] + +DESCRIPTION +----------- + +This allows you to examine the schemas at runtime; while they are fully +documented for the current release in lightning-sql(7), as fields are +added or deprecated, you can use this command to determine what fields +are present. + +If *table* is given, only that table is in the resulting list, otherwise +all tables are listed. + +EXAMPLE JSON REQUEST +------------ +```json +{ + "id": 82, + "method": "listsqlschemas", + "params": { + "table": "offers" + } +} +``` + +EXAMPLE JSON RESPONSE +----- +```json +{ + "schemas": [ + { + "tablename": "offers", + "columns": [ + { + "name": "offer_id", + "type": "BLOB" + }, + { + "name": "active", + "type": "INTEGER" + }, + { + "name": "single_use", + "type": "INTEGER" + }, + { + "name": "bolt12", + "type": "TEXT" + }, + { + "name": "bolt12_unsigned", + "type": "TEXT" + }, + { + "name": "used", + "type": "INTEGER" + }, + { + "name": "label", + "type": "TEXT" + } + ], + "indices": [ + [ + "offer_id" + ] + ] + } + ] +} +``` + +RETURN VALUE +------------ + +[comment]: # (GENERATE-FROM-SCHEMA-START) +On success, an object containing **schemas** is returned. It is an array of objects, where each object contains: + +- **tablename** (string): the name of the table +- **columns** (array of objects): the columns, in database order: + - **name** (string): the name the column + - **type** (string): the SQL type of the column (one of "INTEGER", "BLOB", "TEXT", "REAL") +- **indices** (array of arrays, optional): Any index we created to speed lookups: + - The columns for this index: + - The column name + +[comment]: # (GENERATE-FROM-SCHEMA-END) + +AUTHOR +------ + +Rusty Russell <> is mainly responsible. + +SEE ALSO +-------- + +lightning-sql(7). + +RESOURCES +--------- + +Main web site: +[comment]: # ( SHA256STAMP:3ac985dd8ef6959b327e6e6a79079db3ad51423bc4e469799a12ae74b2e75697) diff --git a/doc/lightning-sql.7.md b/doc/lightning-sql.7.md index f682a00e7..480ca7738 100644 --- a/doc/lightning-sql.7.md +++ b/doc/lightning-sql.7.md @@ -19,6 +19,9 @@ cache `listnodes` and `listchannels`) which then processes the results. It is, however faster for remote access if the result of the query is much smaller than the list commands would be. +Note that queries like "SELECT *" are fragile, as columns will +change across releases; see lightning-listsqlschemas(7). + TREATMENT OF TYPES ------------------ diff --git a/doc/schemas/listsqlschemas.request.json b/doc/schemas/listsqlschemas.request.json new file mode 100644 index 000000000..f12785b1b --- /dev/null +++ b/doc/schemas/listsqlschemas.request.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": [], + "properties": { + "table": { + "type": "string" + } + } +} diff --git a/doc/schemas/listsqlschemas.schema.json b/doc/schemas/listsqlschemas.schema.json new file mode 100644 index 000000000..01143f1b8 --- /dev/null +++ b/doc/schemas/listsqlschemas.schema.json @@ -0,0 +1,67 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "schemas" + ], + "properties": { + "schemas": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "tablename", + "columns" + ], + "properties": { + "tablename": { + "type": "string", + "description": "the name of the table" + }, + "columns": { + "type": "array", + "description": "the columns, in database order", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type" + ], + "properties": { + "name": { + "type": "string", + "description": "the name the column" + }, + "type": { + "type": "string", + "enum": [ + "INTEGER", + "BLOB", + "TEXT", + "REAL" + ], + "description": "the SQL type of the column" + } + } + } + }, + "indices": { + "type": "array", + "description": "Any index we created to speed lookups", + "items": { + "type": "array", + "description": "The columns for this index", + "items": { + "type": "string", + "description": "The column name" + } + } + } + } + } + } + } +} diff --git a/plugins/sql.c b/plugins/sql.c index 9890bac61..bf870d561 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -1001,6 +1001,111 @@ static bool ignore_column(const struct table_desc *td, const jsmntok_t *t) return false; } +static struct command_result *param_tablename(struct command *cmd, + const char *name, + const char *buffer, + const jsmntok_t *tok, + struct table_desc **td) +{ + *td = strmap_getn(&tablemap, buffer + tok->start, + tok->end - tok->start); + if (!*td) + return command_fail_badparam(cmd, name, buffer, tok, + "Unknown table"); + return NULL; +} + +static void json_add_column(struct json_stream *js, + const char *dbname, + const char *sqltypename) +{ + json_object_start(js, NULL); + json_add_string(js, "name", dbname); + json_add_string(js, "type", sqltypename); + json_object_end(js); +} + +static void json_add_columns(struct json_stream *js, + const struct table_desc *td) +{ + for (size_t i = 0; i < tal_count(td->columns); i++) { + if (td->columns[i].sub) { + if (td->columns[i].sub->is_subobject) + json_add_columns(js, td->columns[i].sub); + continue; + } + json_add_column(js, td->columns[i].dbname, + fieldtypemap[td->columns[i].ftype].sqltype); + } +} + +static void json_add_schema(struct json_stream *js, + const struct table_desc *td) +{ + bool have_indices; + + json_object_start(js, NULL); + json_add_string(js, "tablename", td->name); + /* This needs to be an array, not a dictionary, since dicts + * are often treated as unordered, and order is critical! */ + json_array_start(js, "columns"); + if (td->parent) { + json_add_column(js, "row", "INTEGER"); + json_add_column(js, "arrindex", "INTEGER"); + } + json_add_columns(js, td); + json_array_end(js); + + /* Don't print indices entry unless we have an index! */ + have_indices = false; + for (size_t i = 0; i < ARRAY_SIZE(indices); i++) { + if (!streq(indices[i].tablename, td->name)) + continue; + if (!have_indices) { + json_array_start(js, "indices"); + have_indices = true; + } + json_array_start(js, NULL); + for (size_t j = 0; j < ARRAY_SIZE(indices[i].fields); j++) { + if (indices[i].fields[j]) + json_add_string(js, NULL, indices[i].fields[j]); + } + json_array_end(js); + } + if (have_indices) + json_array_end(js); + json_object_end(js); +} + +static bool add_one_schema(const char *member, struct table_desc *td, + struct json_stream *js) +{ + json_add_schema(js, td); + return true; +} + +static struct command_result *json_listsqlschemas(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + struct table_desc *td; + struct json_stream *ret; + + if (!param(cmd, buffer, params, + p_opt("table", param_tablename, &td), + NULL)) + return command_param_failed(); + + ret = jsonrpc_stream_success(cmd); + json_array_start(ret, "schemas"); + if (td) + json_add_schema(ret, td); + else + strmap_iterate(&tablemap, add_one_schema, ret); + json_array_end(ret); + return command_finished(cmd, ret); +} + /* Creates sql statements, initializes table */ static void finish_td(struct plugin *plugin, struct table_desc *td) { @@ -1353,6 +1458,13 @@ static const struct plugin_command commands[] = { { "This is the greatest plugin command ever!", json_sql, }, + { + "listsqlschemas", + "misc", + "Display schemas for internal sql tables, or just {table}", + "This is the greatest plugin command ever!", + json_listsqlschemas, + }, }; static const char *fmt_indexes(const tal_t *ctx, const char *table) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 6a6b43a2c..2b17978b6 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3783,6 +3783,34 @@ def test_sql(node_factory, bitcoind): {'name': 'payment_id', 'type': 'hex'}]}} + sqltypemap = {'string': 'TEXT', + 'boolean': 'INTEGER', + 'u8': 'INTEGER', + 'u16': 'INTEGER', + 'u32': 'INTEGER', + 'u64': 'INTEGER', + 'msat': 'INTEGER', + 'hex': 'BLOB', + 'hash': 'BLOB', + 'txid': 'BLOB', + 'pubkey': 'BLOB', + 'secret': 'BLOB', + 'number': 'REAL', + 'short_channel_id': 'TEXT'} + + # Check schemas match. + for table, schema in expected_schemas.items(): + res = only_one(l2.rpc.listsqlschemas(table)['schemas']) + assert res['tablename'] == table + assert res.get('indices') == schema.get('indices') + sqlcolumns = [{'name': c['name'], 'type': sqltypemap[c['type']]} for c in schema['columns']] + assert res['columns'] == sqlcolumns + + # Make sure we didn't miss any + assert (sorted([s['tablename'] for s in l1.rpc.listsqlschemas()['schemas']]) + == sorted(expected_schemas.keys())) + assert len(l1.rpc.listsqlschemas()['schemas']) == len(expected_schemas) + # Very rough checks of other list commands (make sure l2 has one of each) l2.rpc.offer(1, 'desc') l2.rpc.invoice(1, 'label', 'desc')