From c230291141a2ba74532bac6e134c36139147a134 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:17 +1030 Subject: [PATCH] plugins/sql: rework to parse schemas. This requires us to rename "index" fields, rename fields if we have a sub-object, and create sub-tables if we have an array, and handle the fact that some listX commands don't contain array X (listsendpays contains "payments"). Signed-off-by: Rusty Russell --- plugins/Makefile | 14 ++ plugins/sql.c | 412 ++++++++++++++++++++++++++++++++++++------- tests/test_plugin.py | 33 +++- 3 files changed, 394 insertions(+), 65 deletions(-) diff --git a/plugins/Makefile b/plugins/Makefile index ffbdd0ded..b1a4e8363 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -204,6 +204,20 @@ plugins/fetchinvoice: $(PLUGIN_FETCHINVOICE_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_CO plugins/funder: bitcoin/psbt.o common/psbt_open.o $(PLUGIN_FUNDER_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) +# This covers all the low-level list RPCs which return simple arrays +SQL_LISTRPCS := listchannels listforwards listhtlcs listinvoices listnodes listoffers listpeers listtransactions listsendpays +SQL_LISTRPCS_SCHEMAS := $(foreach l,$(SQL_LISTRPCS),doc/schemas/$l.schema.json) +# We squeeze: +# descriptions (we don't need) +# fields with no members (we don't need) +# whitespace +# We can't simply *remove* fields, since the extra comma left over can +# make invalid JSON. Grr! +# But these simple removals drop us from 100k to 29k. +plugins/sql-schema_gen.h: plugins/Makefile $(SQL_LISTRPCS_SCHEMAS) + @$(call VERBOSE,GEN $@, (SEP=""; echo -n '"{'; for f in $(SQL_LISTRPCS); do echo -n "$$SEP\\\"$$f\\\":"; sed -e s/\"description\":\ *\".\*\"/\"\":\"\"/ -e s/\".*\":\ *{}/\"\":{}/ -e s/\"/\\\\\"/g < doc/schemas/$$f.schema.json | tr -d ' \n'; SEP=","; done; echo '}"') > $@) + +plugins/sql.o: plugins/sql-schema_gen.h plugins/sql: $(PLUGIN_SQL_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) # Generated from PLUGINS definition in plugins/Makefile diff --git a/plugins/sql.c b/plugins/sql.c index e6625cbd0..f07f68cd8 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -1,6 +1,7 @@ /* Brilliant or insane? You decide! */ #include "config.h" #include +#include #include #include #include @@ -9,8 +10,12 @@ #include #include +/* Minimized schemas. C23 #embed, Where Art Thou? */ +static const char schemas[] = + #include "sql-schema_gen.h" + ; + /* TODO: - * 1. Generate from schemas. * 2. Refresh time in API. * 3. Colnames API to return dict. * 4. sql-schemas command. @@ -66,8 +71,12 @@ static const struct fieldtypemap fieldtypemap[] = { }; struct column { - const char *name; + /* We rename some fields to avoid sql keywords! */ + const char *dbname, *jsonname; enum fieldtype ftype; + + /* If this is actually a subtable: */ + struct table_desc *sub; }; struct db_query { @@ -77,16 +86,38 @@ struct db_query { }; struct table_desc { + /* e.g. listpeers */ + const char *cmdname; /* e.g. peers for listpeers */ const char *name; + /* e.g. "payments" for listsendpays */ + const char *arrname; struct column *columns; char *update_stmt; + /* If we are a subtable */ + struct table_desc *parent; + /* Is this a sub object (otherwise, subarray if parent is true) */ + bool is_subobject; + /* function to refresh it. */ + struct command_result *(*refresh)(struct command *cmd, + const struct table_desc *td, + struct db_query *dbq); }; static STRMAP(struct table_desc *) tablemap; static size_t max_dbmem = 500000000; static struct sqlite3 *db; static const char *dbfilename; +static enum fieldtype find_fieldtype(const jsmntok_t *name) +{ + for (size_t i = 0; i < ARRAY_SIZE(fieldtypemap); i++) { + if (json_tok_streq(schemas, name, fieldtypemap[i].name)) + return i; + } + errx(1, "Unknown JSON type %.*s", + name->end - name->start, schemas + name->start); +} + static struct sqlite3 *sqlite_setup(struct plugin *plugin) { int err; @@ -165,6 +196,9 @@ static int sqlite_authorize(void *dbq_, int code, dbq->authfail = tal_fmt(dbq, "Unknown table %s", a); return SQLITE_DENY; } + /* If it has a parent, we refresh that instead */ + while (td->parent) + td = td->parent; if (!has_table_desc(dbq->tables, td)) tal_arr_expand(&dbq->tables, td); return SQLITE_OK; @@ -284,6 +318,13 @@ static struct command_result *one_refresh_done(struct command *cmd, return refresh_tables(cmd, dbq); } +/* Mutual recursion */ +static struct command_result *process_json_list(struct command *cmd, + const char *buf, + const jsmntok_t *arr, + const u64 *rowid, + const struct table_desc *td); + /* Returns NULL on success, otherwise has failed cmd. */ static struct command_result *process_json_obj(struct command *cmd, const char *buf, @@ -295,20 +336,47 @@ static struct command_result *process_json_obj(struct command *cmd, sqlite3_stmt *stmt) { int err; + u64 parent_rowid; + + /* Subtables have row, arrindex as first two columns. */ + if (rowid) { + sqlite3_bind_int64(stmt, (*sqloff)++, *rowid); + sqlite3_bind_int64(stmt, (*sqloff)++, row); + } /* FIXME: This is O(n^2): hash td->columns and look up the other way. */ for (size_t i = 0; i < tal_count(td->columns); i++) { const struct column *col = &td->columns[i]; const jsmntok_t *coltok; + if (col->sub) { + struct command_result *ret; + /* Handle sub-tables below: we need rowid! */ + if (!col->sub->is_subobject) + continue; + + coltok = json_get_member(buf, t, col->jsonname); + ret = process_json_obj(cmd, buf, coltok, col->sub, row, NULL, + sqloff, stmt); + if (ret) + return ret; + continue; + } + + /* This can happen if subobject does not exist in output! */ if (!t) coltok = NULL; else - coltok = json_get_member(buf, t, col->name); + coltok = json_get_member(buf, t, col->jsonname); - if (!coltok) + if (!coltok) { + if (td->parent) + plugin_log(cmd->plugin, LOG_DBG, + "Did not find json %s for %s in %.*s", + col->jsonname, td->name, + t ? json_tok_full_len(t) : 4, t ? json_tok_full(buf, t): "NULL"); sqlite3_bind_null(stmt, (*sqloff)++); - else { + } else { u64 val64; struct amount_msat valmsat; u8 *valhex; @@ -386,6 +454,10 @@ static struct command_result *process_json_obj(struct command *cmd, } } + /* Sub objects get folded into parent's SQL */ + if (td->parent && td->is_subobject) + return NULL; + err = sqlite3_step(stmt); if (err != SQLITE_DONE) { return command_fail(cmd, LIGHTNINGD, @@ -394,16 +466,37 @@ static struct command_result *process_json_obj(struct command *cmd, row, sqlite3_errmsg(db)); } + + /* Now we have rowid, we can insert into any subtables. */ + parent_rowid = sqlite3_last_insert_rowid(db); + for (size_t i = 0; i < tal_count(td->columns); i++) { + const struct column *col = &td->columns[i]; + const jsmntok_t *coltok; + struct command_result *ret; + + if (!col->sub || col->sub->is_subobject) + continue; + + coltok = json_get_member(buf, t, col->jsonname); + if (!coltok) + continue; + + ret = process_json_list(cmd, buf, coltok, &parent_rowid, col->sub); + if (ret) + return ret; + } return NULL; } +/* A list, such as in the top-level reply, or for a sub-table */ static struct command_result *process_json_list(struct command *cmd, const char *buf, - const jsmntok_t *result, + const jsmntok_t *arr, + const u64 *rowid, const struct table_desc *td) { size_t i; - const jsmntok_t *t, *arr = json_get_member(buf, result, td->name); + const jsmntok_t *t; int err; sqlite3_stmt *stmt; struct command_result *ret = NULL; @@ -415,10 +508,10 @@ static struct command_result *process_json_list(struct command *cmd, sqlite3_errmsg(db)); } - json_for_each_arr(i, t, arr) { + json_for_each_arr(i, t, arr) { /* sqlite3 columns are 1-based! */ size_t off = 1; - ret = process_json_obj(cmd, buf, t, td, i, NULL, &off, stmt); + ret = process_json_obj(cmd, buf, t, td, i, rowid, &off, stmt); if (ret) break; sqlite3_reset(stmt); @@ -427,6 +520,17 @@ static struct command_result *process_json_list(struct command *cmd, return ret; } +/* Process top-level JSON result object */ +static struct command_result *process_json_result(struct command *cmd, + const char *buf, + const jsmntok_t *result, + const struct table_desc *td) +{ + return process_json_list(cmd, buf, + json_get_member(buf, result, td->arrname), + NULL, td); +} + static struct command_result *default_list_done(struct command *cmd, const char *buf, const jsmntok_t *result, @@ -445,7 +549,7 @@ static struct command_result *default_list_done(struct command *cmd, td->name, errmsg); } - ret = process_json_list(cmd, buf, result, td); + ret = process_json_result(cmd, buf, result, td); if (ret) return ret; @@ -457,8 +561,7 @@ static struct command_result *default_refresh(struct command *cmd, struct db_query *dbq) { struct out_req *req; - req = jsonrpc_request_start(cmd->plugin, cmd, - tal_fmt(tmpctx, "list%s", td->name), + req = jsonrpc_request_start(cmd->plugin, cmd, td->cmdname, default_list_done, forward_error, dbq); return send_outreq(cmd->plugin, req); @@ -473,7 +576,7 @@ static struct command_result *refresh_tables(struct command *cmd, return refresh_complete(cmd, dbq); td = dbq->tables[0]; - return default_refresh(cmd, td, dbq); + return td->refresh(cmd, dbq->tables[0], dbq); } static struct command_result *json_sql(struct command *cmd, @@ -513,64 +616,74 @@ static struct command_result *json_sql(struct command *cmd, return refresh_tables(cmd, dbq); } -static void init_tablemap(struct plugin *plugin) +static bool ignore_column(const struct table_desc *td, const jsmntok_t *t) +{ + /* We don't use peers.log, since it'll always be empty unless we were to + * ask for it in listpeers, and it's not very useful. */ + if (streq(td->name, "peers") && json_tok_streq(schemas, t, "log")) + return true; + /* FIXME: peers.netaddr is an array of strings, which we don't handle. */ + if (streq(td->name, "peers") && json_tok_streq(schemas, t, "netaddr")) + return true; + /* FIXME: peers.channels.features is an array of strings, which we don't handle. */ + if (streq(td->name, "peers_channels") && json_tok_streq(schemas, t, "features")) + return true; + if (streq(td->name, "peers_channels") && json_tok_streq(schemas, t, "status")) + return true; + return false; +} + +/* Creates sql statements, initializes table, adds to tablemap */ +static void finish_td(struct plugin *plugin, struct table_desc *td) { - struct table_desc *td; char *create_stmt; int err; char *errmsg; - struct column col; + const char *sep = ""; - strmap_init(&tablemap); - - /* FIXME: Load from schemas! */ - td = tal(NULL, struct table_desc); - td->name = "forwards"; - td->columns = tal_arr(td, struct column, 0); - col.name = "in_htlc_id"; - col.ftype = FIELD_U64; - tal_arr_expand(&td->columns, col); - col.name = "in_channel"; - col.ftype = FIELD_SCID; - tal_arr_expand(&td->columns, col); - col.name = "in_msat"; - col.ftype = FIELD_MSAT; - tal_arr_expand(&td->columns, col); - col.name = "status"; - col.ftype = FIELD_STRING; - tal_arr_expand(&td->columns, col); - col.name = "received_time"; - col.ftype = FIELD_NUMBER; - tal_arr_expand(&td->columns, col); - col.name = "out_channel"; - col.ftype = FIELD_SCID; - tal_arr_expand(&td->columns, col); - col.name = "out_htlc_id"; - col.ftype = FIELD_U64; - tal_arr_expand(&td->columns, col); - col.name = "style"; - col.ftype = FIELD_STRING; - tal_arr_expand(&td->columns, col); - col.name = "fee_msat"; - col.ftype = FIELD_MSAT; - tal_arr_expand(&td->columns, col); - col.name = "out_msat"; - col.ftype = FIELD_MSAT; - tal_arr_expand(&td->columns, col); - col.name = "resolved_time"; - col.ftype = FIELD_NUMBER; - tal_arr_expand(&td->columns, col); + /* subobject are separate at JSON level, folded at db level! */ + if (td->is_subobject) + return; /* FIXME: Primary key from schema? */ create_stmt = tal_fmt(tmpctx, "CREATE TABLE %s (", td->name); td->update_stmt = tal_fmt(td, "INSERT INTO %s VALUES (", td->name); + + /* If we're a child array, we reference the parent column */ + if (td->parent) { + tal_append_fmt(&create_stmt, + "row INTEGER REFERENCES %s(rowid) ON DELETE CASCADE," + " arrindex INTEGER", + td->parent->name); + tal_append_fmt(&td->update_stmt, "?,?"); + sep = ","; + } + for (size_t i = 0; i < tal_count(td->columns); i++) { - tal_append_fmt(&td->update_stmt, "%s?", - i == 0 ? "" : ","); + const struct column *col = &td->columns[i]; + + if (col->sub) { + /* sub-arrays are a completely separate table. */ + if (!col->sub->is_subobject) + continue; + /* sub-objects are folded into this table. */ + for (size_t j = 0; j < tal_count(col->sub->columns); j++) { + const struct column *subcol = &col->sub->columns[j]; + tal_append_fmt(&td->update_stmt, "%s?", sep); + tal_append_fmt(&create_stmt, "%s%s %s", + sep, + subcol->dbname, + fieldtypemap[subcol->ftype].sqltype); + sep = ","; + } + continue; + } + tal_append_fmt(&td->update_stmt, "%s?", sep); tal_append_fmt(&create_stmt, "%s%s %s", - i == 0 ? "" : ",", - td->columns[i].name, - fieldtypemap[td->columns[i].ftype].sqltype); + sep, + col->dbname, + fieldtypemap[col->ftype].sqltype); + sep = ","; } tal_append_fmt(&create_stmt, ");"); tal_append_fmt(&td->update_stmt, ");"); @@ -580,6 +693,185 @@ static void init_tablemap(struct plugin *plugin) plugin_err(plugin, "Could not create %s: %s", td->name, errmsg); strmap_add(&tablemap, td->name, td); + + /* Now do any children */ + for (size_t i = 0; i < tal_count(td->columns); i++) { + const struct column *col = &td->columns[i]; + if (col->sub) + finish_td(plugin, col->sub); + } +} + +/* Don't use SQL keywords as column names: sure, you can use quotes, + * but it's a PITA. */ +static const char *db_column_name(const tal_t *ctx, + const struct table_desc *td, + const jsmntok_t *nametok) +{ + const char *name = json_strdup(tmpctx, schemas, nametok); + + if (streq(name, "index")) + name = tal_strdup(tmpctx, "idx"); + + /* Prepend td->name to make column unique in table. */ + if (td->is_subobject) + return tal_fmt(ctx, "%s_%s", td->cmdname, name); + + return tal_steal(ctx, name); +} + +/* Remove 'list', turn - into _ in name */ +static const char *db_table_name(const tal_t *ctx, const char *cmdname) +{ + const char *list = strstr(cmdname, "list"); + char *ret = tal_arr(ctx, char, strlen(cmdname) + 1), *dst = ret; + const char *src = cmdname; + + while (*src) { + if (src == list) + src += strlen("list"); + else if (cisalnum(*src)) + *(dst++) = *(src++); + else { + (*dst++) = '_'; + src++; + } + } + *dst = '\0'; + return ret; +} + +static struct table_desc *new_table_desc(struct table_desc *parent, + const jsmntok_t *cmd, + const jsmntok_t *arrname, + bool is_subobject) +{ + struct table_desc *td; + const char *name; + + td = tal(parent, struct table_desc); + td->cmdname = json_strdup(td, schemas, cmd); + name = db_table_name(tmpctx, td->cmdname); + if (!parent) + td->name = tal_steal(td, name); + else + td->name = tal_fmt(td, "%s_%s", parent->name, name); + td->parent = parent; + td->is_subobject = is_subobject; + td->arrname = json_strdup(td, schemas, arrname); + td->columns = tal_arr(td, struct column, 0); + td->refresh = default_refresh; + return td; +} + +static bool find_column(const struct table_desc *td, + const char *dbname) +{ + for (size_t i = 0; i < tal_count(td->columns); i++) { + if (streq(td->columns[i].dbname, dbname)) + return true; + } + return false; +} + +/* Recursion */ +static void add_table_object(struct table_desc *td, const jsmntok_t *tok); + +static void add_table_properties(struct table_desc *td, + const jsmntok_t *properties) +{ + const jsmntok_t *t; + size_t i; + + json_for_each_obj(i, t, properties) { + const jsmntok_t *type; + struct column col; + + if (ignore_column(td, t)) + continue; + type = json_get_member(schemas, t+1, "type"); + /* Stub properties don't have types, it should exist in + * another branch with actual types, so ignore this */ + if (!type) + continue; + if (json_tok_streq(schemas, type, "array")) { + const jsmntok_t *items; + + items = json_get_member(schemas, t+1, "items"); + type = json_get_member(schemas, items, "type"); + assert(json_tok_streq(schemas, type, "object")); + + col.sub = new_table_desc(td, t, t, false); + add_table_object(col.sub, items); + } else if (json_tok_streq(schemas, type, "object")) { + col.sub = new_table_desc(td, t, t, true); + add_table_object(col.sub, t+1); + } else { + col.ftype = find_fieldtype(type); + col.sub = NULL; + } + col.dbname = db_column_name(td->columns, td, t); + /* Some schemas repeat, assume they're the same */ + if (find_column(td, col.dbname)) { + tal_free(col.dbname); + } else { + col.jsonname = json_strdup(td->columns, schemas, t); + tal_arr_expand(&td->columns, col); + } + } +} + +/* tok is the JSON schema member for an object */ +static void add_table_object(struct table_desc *td, const jsmntok_t *tok) +{ + const jsmntok_t *t, *properties, *allof, *cond; + size_t i; + + /* This might not exist inside allOf, for example */ + properties = json_get_member(schemas, tok, "properties"); + if (properties) + add_table_properties(td, properties); + + allof = json_get_member(schemas, tok, "allOf"); + if (allof) { + json_for_each_arr(i, t, allof) + add_table_object(td, t); + } + /* We often find interesting things in then and else branches! */ + cond = json_get_member(schemas, tok, "then"); + if (cond) + add_table_object(td, cond); + cond = json_get_member(schemas, tok, "else"); + if (cond) + add_table_object(td, cond); +} + +static void init_tablemap(struct plugin *plugin) +{ + const jsmntok_t *toks, *t; + size_t i; + + strmap_init(&tablemap); + + toks = json_parse_simple(tmpctx, schemas, strlen(schemas)); + json_for_each_obj(i, t, toks) { + struct table_desc *td; + const jsmntok_t *cmd, *items, *type; + + /* First member of properties object is command. */ + cmd = json_get_member(schemas, t+1, "properties") + 1; + + /* We assume it's an object containing an array of objects */ + items = json_get_member(schemas, cmd + 1, "items"); + type = json_get_member(schemas, items, "type"); + assert(json_tok_streq(schemas, type, "object")); + + td = new_table_desc(NULL, t, cmd, false); + tal_steal(plugin, td); + add_table_object(td, items); + + finish_td(plugin, td); + } } #if DEVELOPER diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 2e20dcd8d..d1790e002 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3278,7 +3278,9 @@ def test_block_added_notifications(node_factory, bitcoind): def test_sql(node_factory, bitcoind): - l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True) + l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True, + opts={'experimental-offers': None, + 'sqlfilename': 'sql.sqlite3'}) ret = l2.rpc.sql("SELECT * FROM forwards;") assert ret == {'rows': []} @@ -3286,11 +3288,32 @@ def test_sql(node_factory, bitcoind): # This should create a forward through l2 l1.rpc.pay(l3.rpc.invoice(amount_msat=12300, label='inv1', description='description')['bolt11']) + # Very rough checks of other list commands: + ret = l1.rpc.sql("SELECT * FROM htlcs;") + assert len(only_one(ret['rows'])) == 7 + + ret = l3.rpc.sql("SELECT * FROM invoices;") + assert len(only_one(ret['rows'])) == 14 + + ret = l3.rpc.sql("SELECT * FROM nodes;") + assert len(ret['rows']) == 3 + assert len(ret['rows'][0]) == 11 + + ret = l3.rpc.sql("SELECT * FROM peers;") + assert len(only_one(ret['rows'])) == 4 + + l3.rpc.offer(1, 'desc') + ret = l3.rpc.sql("SELECT * FROM offers;") + assert len(only_one(ret['rows'])) == 6 + + ret = l1.rpc.sql("SELECT * FROM sendpays;") + assert len(only_one(ret['rows'])) == 15 + + ret = l3.rpc.sql("SELECT * FROM transactions;") + assert len(only_one(ret['rows'])) == 6 + ret = l2.rpc.sql("SELECT in_htlc_id,out_msat,status,out_htlc_id FROM forwards;") - assert only_one(ret['rows'])[0] == 0 - assert only_one(ret['rows'])[1] == 12300 - assert only_one(ret['rows'])[2] == 'settled' - assert only_one(ret['rows'])[3] == 0 + assert only_one(ret['rows']) == [0, 12300, 'settled', 0] with pytest.raises(RpcError, match='Unauthorized'): l2.rpc.sql("DELETE FROM forwards;")