diff --git a/doc/lightning-bkpr-listaccountevents.7.md b/doc/lightning-bkpr-listaccountevents.7.md index b2b393aa7..4e132eedf 100644 --- a/doc/lightning-bkpr-listaccountevents.7.md +++ b/doc/lightning-bkpr-listaccountevents.7.md @@ -38,6 +38,7 @@ If **type** is "chain": - **origin** (string, optional): The account this movement originated from - **payment_id** (hex, optional): lightning payment identifier. For an htlc, this will be the preimage. - **txid** (txid, optional): The txid of the transaction that created this event + - **description** (string, optional): The description of this event If **type** is "onchain_fee": - **txid** (txid): The txid of the transaction that created this event @@ -65,4 +66,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:dd72cc73e685daa6877984be8edede76dfec2f9d85df9a88ab1b031a93b20549) +[comment]: # ( SHA256STAMP:f8538b1d1e6cda7cd801690e5c09741c8a843b27cc922065598914516c16d2b3) diff --git a/doc/lightning-bkpr-listincome.7.md b/doc/lightning-bkpr-listincome.7.md index d7a3d87da..dfc3647e1 100644 --- a/doc/lightning-bkpr-listincome.7.md +++ b/doc/lightning-bkpr-listincome.7.md @@ -33,6 +33,7 @@ On success, an object containing **income_events** is returned. It is an array - **debit_msat** (msat): Amount spent (expenses) - **currency** (string): human-readable bech32 part for this coin type - **timestamp** (u32): Timestamp this event was recorded by the node. For consolidated events such as onchain_fees, the most recent timestamp +- **description** (string, optional): More information about this event. If a `invoice` type, typically the bolt11/bolt12 description - **outpoint** (string, optional): The txid:outnum for this event, if applicable - **txid** (txid, optional): The txid of the transaction that created this event, if applicable - **payment_id** (hex, optional): lightning payment identifier. For an htlc, this will be the preimage. @@ -55,4 +56,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:400ac5e6719a7ae5ec7078a2cd220d91ab7e66ad45f08b46257e6ec04dcdeb4c) +[comment]: # ( SHA256STAMP:ab8508af0f40587c5a804f6981591564fe2d18b4fe3fbe7793e6a489607f7e0a) diff --git a/doc/schemas/bkpr-listaccountevents.schema.json b/doc/schemas/bkpr-listaccountevents.schema.json index 2a72cbf8e..ca0fa6c87 100644 --- a/doc/schemas/bkpr-listaccountevents.schema.json +++ b/doc/schemas/bkpr-listaccountevents.schema.json @@ -95,6 +95,10 @@ "txid": { "type": "txid", "description": "The txid of the transaction that created this event" + }, + "description": { + "type": "string", + "description": "The description of this event" } }, "required": [ @@ -124,6 +128,7 @@ "debit_msat": {}, "currency": {}, "timestamp": {}, + "description": {}, "txid": { "type": "txid", "description": "The txid of the transaction that created this event" @@ -155,6 +160,7 @@ "debit_msat": {}, "currency": {}, "timestamp": {}, + "description": {}, "fees_msat": { "type": "msat", "description": "Amount paid in fees" diff --git a/doc/schemas/bkpr-listincome.schema.json b/doc/schemas/bkpr-listincome.schema.json index b572bf80b..147dc484c 100644 --- a/doc/schemas/bkpr-listincome.schema.json +++ b/doc/schemas/bkpr-listincome.schema.json @@ -44,6 +44,10 @@ "type": "u32", "description": "Timestamp this event was recorded by the node. For consolidated events such as onchain_fees, the most recent timestamp" }, + "description": { + "type": "string", + "description": "More information about this event. If a `invoice` type, typically the bolt11/bolt12 description" + }, "outpoint": { "type": "string", "description": "The txid:outnum for this event, if applicable" diff --git a/plugins/bkpr/Makefile b/plugins/bkpr/Makefile index 7e7730e79..ac30fafe2 100644 --- a/plugins/bkpr/Makefile +++ b/plugins/bkpr/Makefile @@ -37,7 +37,7 @@ PLUGIN_ALL_HEADER += $(BOOKKEEPER_HEADER) C_PLUGINS += plugins/bookkeeper PLUGINS += plugins/bookkeeper -plugins/bookkeeper: bitcoin/chainparams.o common/coin_mvt.o $(BOOKKEEPER_OBJS) $(PLUGIN_LIB_OBJS) $(JSMN_OBJTS) $(PLUGIN_COMMON_OBJS) $(WIRE_OBJS) $(DB_OBJS) +plugins/bookkeeper: bitcoin/chainparams.o common/bolt12.o common/bolt12_merkle.o $(BOOKKEEPER_OBJS) $(PLUGIN_LIB_OBJS) $(JSMN_OBJTS) $(PLUGIN_COMMON_OBJS) $(WIRE_OBJS) $(DB_OBJS) # The following files contain SQL-annotated statements that we need to extact BOOKKEEPER_SQL_FILES := \ diff --git a/plugins/bkpr/bookkeeper.c b/plugins/bkpr/bookkeeper.c index cd20bdf4d..5321fd87e 100644 --- a/plugins/bkpr/bookkeeper.c +++ b/plugins/bkpr/bookkeeper.c @@ -2,7 +2,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -623,6 +626,7 @@ static bool new_missed_channel_account(struct command *cmd, chain_ev->payment_id = NULL; chain_ev->ignored = false; chain_ev->stealable = false; + chain_ev->desc = NULL; /* Update the account info too */ tags = tal_arr(chain_ev, enum mvt_tag, 1); @@ -798,7 +802,7 @@ static struct command_result *log_error(struct command *cmd, void *arg UNNEEDED) { plugin_log(cmd->plugin, LOG_BROKEN, - "error calling `listpeers`: %.*s", + "error calling rpc: %.*s", json_tok_full_len(error), json_tok_full(buf, error)); @@ -898,70 +902,6 @@ static char *do_account_close_checks(const tal_t *ctx, return NULL; } -struct event_info { - struct chain_event *ev; - struct account *acct; -}; - -static struct command_result * -listpeers_done(struct command *cmd, const char *buf, - const jsmntok_t *result, struct event_info *info) -{ - struct acct_balance **balances, *bal; - struct amount_msat credit_diff, debit_diff; - const char *err; - /* Make sure to clean up when we're done */ - tal_steal(cmd, info); - - if (new_missed_channel_account(cmd, buf, result, - info->acct, - info->ev->currency, - info->ev->timestamp)) { - db_begin_transaction(db); - err = account_get_balance(tmpctx, db, info->acct->name, - false, false, &balances); - db_commit_transaction(db); - - if (err) - plugin_err(cmd->plugin, err); - - /* FIXME: multiple currencies per account? */ - if (tal_count(balances) > 0) - bal = balances[0]; - else { - bal = tal(balances, struct acct_balance); - bal->credit = AMOUNT_MSAT(0); - bal->debit = AMOUNT_MSAT(0); - } - assert(tal_count(balances) == 1); - - /* The expected current balance is zero, since - * we just got the channel close event */ - err = msat_find_diff(AMOUNT_MSAT(0), - bal->credit, - bal->debit, - &credit_diff, &debit_diff); - if (err) - plugin_err(cmd->plugin, err); - - log_journal_entry(info->acct, - info->ev->currency, - info->ev->timestamp - 1, - credit_diff, debit_diff); - } else - plugin_log(cmd->plugin, LOG_BROKEN, - "Unable to find account %s in listpeers", - info->acct->name); - - /* Maybe mark acct as onchain resolved */ - err = do_account_close_checks(cmd, info->ev, info->acct); - if (err) - plugin_err(cmd->plugin, err); - - return notification_handled(cmd); -} - - static struct command_result *json_balance_snapshot(struct command *cmd, const char *buf, const jsmntok_t *params) @@ -1144,6 +1084,242 @@ static struct command_result *json_balance_snapshot(struct command *cmd, return notification_handled(cmd); } +/* Returns true if "fatal" error, otherwise just a normal error */ +static char *fetch_out_desc_invstr(const tal_t *ctx, const char *buf, + const jsmntok_t *tok, char **err) +{ + char *bolt, *desc, *fail; + + /* It's a bolt11! Parse it out to a desc */ + if (!json_scan(ctx, buf, tok, "{bolt11:%}", + JSON_SCAN_TAL(ctx, json_strdup, &bolt))) { + struct bolt11 *bolt11; + u5 *sigdata; + struct sha256 hash; + bool have_n; + + bolt11 = bolt11_decode_nosig(ctx, bolt, + /* No desc/features/chain checks */ + NULL, NULL, NULL, + &hash, &sigdata, &have_n, + &fail); + + if (bolt11) { + if (bolt11->description) + desc = tal_strdup(ctx, bolt11->description); + else if (bolt11->description_hash) + desc = tal_fmt(ctx, "%s", + type_to_string(ctx, + struct sha256, + bolt11->description_hash)); + else + desc = NULL; + } else { + *err = tal_fmt(ctx, "failed to parse bolt11 %s: %s", + bolt, fail); + return NULL; + } + } else if (!json_scan(ctx, buf, tok, "{bolt12:%}", + JSON_SCAN_TAL(ctx, json_strdup, &bolt))) { + struct tlv_invoice *bolt12; + + bolt12 = invoice_decode_nosig(ctx, bolt, strlen(bolt), + /* No features/chain checks */ + NULL, NULL, + &fail); + if (!bolt12) { + *err = tal_fmt(ctx, "failed to parse" + " bolt12 %s: %s", + bolt, fail); + return NULL; + } + + if (bolt12->description) + desc = tal_strndup(ctx, + cast_signed(char *, bolt12->description), + tal_bytelen(bolt12->description)); + else + desc = NULL; + } else + desc = NULL; + + *err = NULL; + return desc; +} + +static struct command_result * +listinvoice_done(struct command *cmd, const char *buf, + const jsmntok_t *result, struct sha256 *payment_hash) +{ + size_t i; + const jsmntok_t *inv_arr_tok, *inv_tok; + const char *desc; + inv_arr_tok = json_get_member(buf, result, "invoices"); + assert(inv_arr_tok->type == JSMN_ARRAY); + + desc = NULL; + json_for_each_arr(i, inv_tok, inv_arr_tok) { + char *err; + + /* Found desc in "description" */ + if (!json_scan(cmd, buf, inv_tok, "{description:%}", + JSON_SCAN_TAL(cmd, json_strdup, &desc))) + break; + + /* if 'description' doesn't exist, try bolt11/bolt12 */ + desc = fetch_out_desc_invstr(cmd, buf, inv_tok, &err); + if (desc || err) { + if (err) + plugin_log(cmd->plugin, + LOG_BROKEN, "%s", err); + break; + } + } + + if (desc) { + db_begin_transaction(db); + add_payment_hash_desc(db, payment_hash, desc); + db_commit_transaction(db); + } else + plugin_log(cmd->plugin, LOG_DBG, + "listinvoices:" + " description/bolt11/bolt12" + " not found (%.*s)", + result->end - result->start, buf); + + return notification_handled(cmd); +} + +static struct command_result * +listsendpays_done(struct command *cmd, const char *buf, + const jsmntok_t *result, struct sha256 *payment_hash) +{ + size_t i; + const jsmntok_t *pays_arr_tok, *pays_tok; + const char *desc; + pays_arr_tok = json_get_member(buf, result, "payments"); + assert(pays_arr_tok->type == JSMN_ARRAY); + + /* Did we find a matching entry? */ + desc = NULL; + json_for_each_arr(i, pays_tok, pays_arr_tok) { + char *err; + + desc = fetch_out_desc_invstr(cmd, buf, pays_tok, &err); + if (desc || err) { + if (err) + plugin_log(cmd->plugin, + LOG_BROKEN, "%s", err); + break; + } + } + + if (desc) { + db_begin_transaction(db); + add_payment_hash_desc(db, payment_hash, desc); + db_commit_transaction(db); + } else + plugin_log(cmd->plugin, LOG_DBG, + "listpays: bolt11/bolt12 not found:" + "(%.*s)", + result->end - result->start, buf); + + return notification_handled(cmd); +} + +static struct command_result *lookup_invoice_desc(struct command *cmd, + struct amount_msat credit, + struct amount_msat debit, + struct sha256 *payment_hash) +{ + struct out_req *req; + + if (!amount_msat_zero(credit)) + req = jsonrpc_request_start(cmd->plugin, cmd, + "listinvoices", + listinvoice_done, + log_error, + payment_hash); + else + req = jsonrpc_request_start(cmd->plugin, cmd, + "listsendpays", + listsendpays_done, + log_error, + payment_hash); + + json_add_sha256(req->js, "payment_hash", payment_hash); + send_outreq(cmd->plugin, req); + return command_still_pending(cmd); +} + +struct event_info { + struct chain_event *ev; + struct account *acct; +}; + +static struct command_result * +listpeers_done(struct command *cmd, const char *buf, + const jsmntok_t *result, struct event_info *info) +{ + struct acct_balance **balances, *bal; + struct amount_msat credit_diff, debit_diff; + const char *err; + /* Make sure to clean up when we're done */ + tal_steal(cmd, info); + + if (new_missed_channel_account(cmd, buf, result, + info->acct, + info->ev->currency, + info->ev->timestamp)) { + db_begin_transaction(db); + err = account_get_balance(tmpctx, db, info->acct->name, + false, false, &balances); + db_commit_transaction(db); + + if (err) + plugin_err(cmd->plugin, err); + + /* FIXME: multiple currencies per account? */ + if (tal_count(balances) > 0) + bal = balances[0]; + else { + bal = tal(balances, struct acct_balance); + bal->credit = AMOUNT_MSAT(0); + bal->debit = AMOUNT_MSAT(0); + } + assert(tal_count(balances) == 1); + + /* The expected current balance is zero, since + * we just got the channel close event */ + err = msat_find_diff(AMOUNT_MSAT(0), + bal->credit, + bal->debit, + &credit_diff, &debit_diff); + if (err) + plugin_err(cmd->plugin, err); + + log_journal_entry(info->acct, + info->ev->currency, + info->ev->timestamp - 1, + credit_diff, debit_diff); + } else + plugin_log(cmd->plugin, LOG_BROKEN, + "Unable to find account %s in listpeers", + info->acct->name); + + /* Maybe mark acct as onchain resolved */ + err = do_account_close_checks(cmd, info->ev, info->acct); + if (err) + plugin_err(cmd->plugin, err); + + if (info->ev->payment_id && + streq(info->ev->tag, mvt_tag_str(INVOICE))) + return lookup_invoice_desc(cmd, info->ev->credit, + info->ev->debit, + info->ev->payment_id); + + return notification_handled(cmd); +} static struct command_result * parse_and_log_chain_move(struct command *cmd, const char *buf, @@ -1153,7 +1329,8 @@ parse_and_log_chain_move(struct command *cmd, const struct amount_msat debit, const char *coin_type STEALS, const u64 timestamp, - const enum mvt_tag *tags) + const enum mvt_tag *tags, + const char *desc) { struct chain_event *e = tal(cmd, struct chain_event); struct sha256 *payment_hash = tal(cmd, struct sha256); @@ -1247,6 +1424,7 @@ parse_and_log_chain_move(struct command *cmd, e->currency = tal_steal(e, coin_type); e->timestamp = timestamp; e->tag = mvt_tag_str(tags[0]); + e->desc = tal_steal(e, desc); e->ignored = false; e->stealable = false; @@ -1255,7 +1433,6 @@ parse_and_log_chain_move(struct command *cmd, e->stealable |= tags[i] == STEALABLE; } - db_begin_transaction(db); acct = find_account(cmd, db, acct_name); @@ -1349,6 +1526,18 @@ parse_and_log_chain_move(struct command *cmd, if (err) plugin_err(cmd->plugin, err); + /* Check for invoice desc data, necessary */ + if (e->payment_id) { + for (size_t i = 0; i < tal_count(tags); i++) { + if (tags[i] != INVOICE) + continue; + + return lookup_invoice_desc(cmd, e->credit, + e->debit, + e->payment_id); + } + } + return notification_handled(cmd);; } @@ -1361,7 +1550,8 @@ parse_and_log_channel_move(struct command *cmd, const struct amount_msat debit, const char *coin_type STEALS, const u64 timestamp, - const enum mvt_tag *tags) + const enum mvt_tag *tags, + const char *desc) { struct channel_event *e = tal(cmd, struct channel_event); struct account *acct; @@ -1397,6 +1587,7 @@ parse_and_log_channel_move(struct command *cmd, e->currency = tal_steal(e, coin_type); e->timestamp = timestamp; e->tag = mvt_tag_str(tags[0]); + e->desc = tal_steal(e, desc); /* Go find the account for this event */ db_begin_transaction(db); @@ -1410,6 +1601,18 @@ parse_and_log_channel_move(struct command *cmd, log_channel_event(db, acct, e); db_commit_transaction(db); + /* Check for invoice desc data, necessary */ + if (e->payment_id) { + for (size_t i = 0; i < tal_count(tags); i++) { + if (tags[i] != INVOICE) + continue; + + return lookup_invoice_desc(cmd, e->credit, + e->debit, + e->payment_id); + } + } + return notification_handled(cmd); } @@ -1434,9 +1637,9 @@ static char *parse_tags(const tal_t *ctx, return NULL; } -static struct command_result * json_coin_moved(struct command *cmd, - const char *buf, - const jsmntok_t *params) +static struct command_result *json_coin_moved(struct command *cmd, + const char *buf, + const jsmntok_t *params) { const char *err, *mvt_type, *acct_name, *coin_type; u32 version; @@ -1490,13 +1693,15 @@ static struct command_result * json_coin_moved(struct command *cmd, if (streq(mvt_type, CHAIN_MOVE)) return parse_and_log_chain_move(cmd, buf, params, acct_name, credit, debit, - coin_type, timestamp, tags); + coin_type, timestamp, tags, + NULL); assert(streq(mvt_type, CHANNEL_MOVE)); return parse_and_log_channel_move(cmd, buf, params, acct_name, credit, debit, - coin_type, timestamp, tags); + coin_type, timestamp, tags, + NULL); } const struct plugin_notification notifs[] = { diff --git a/plugins/bkpr/chain_event.c b/plugins/bkpr/chain_event.c index 633aedb1d..639930333 100644 --- a/plugins/bkpr/chain_event.c +++ b/plugins/bkpr/chain_event.c @@ -22,5 +22,7 @@ void json_add_chain_event(struct json_stream *out, struct chain_event *ev) json_add_sha256(out, "payment_id", ev->payment_id); json_add_u64(out, "timestamp", ev->timestamp); json_add_u32(out, "blockheight", ev->blockheight); + if (ev->desc) + json_add_string(out, "description", ev->desc); json_object_end(out); } diff --git a/plugins/bkpr/chain_event.h b/plugins/bkpr/chain_event.h index 0a2771c51..3fea8f29d 100644 --- a/plugins/bkpr/chain_event.h +++ b/plugins/bkpr/chain_event.h @@ -60,6 +60,9 @@ struct chain_event { /* Sometimes chain events resolve payments */ struct sha256 *payment_id; + + /* Desc of event (maybe useful for printing notes) */ + const char *desc; }; void json_add_chain_event(struct json_stream *out, diff --git a/plugins/bkpr/channel_event.c b/plugins/bkpr/channel_event.c index f127c6b7c..3952a0b9f 100644 --- a/plugins/bkpr/channel_event.c +++ b/plugins/bkpr/channel_event.c @@ -26,6 +26,7 @@ struct channel_event *new_channel_event(const tal_t *ctx, ev->payment_id = tal_steal(ev, payment_id); ev->part_id = part_id; ev->timestamp = timestamp; + ev->desc = NULL; return ev; } @@ -47,5 +48,7 @@ void json_add_channel_event(struct json_stream *out, json_add_u32(out, "part_id", ev->part_id); } json_add_u64(out, "timestamp", ev->timestamp); + if (ev->desc) + json_add_string(out, "description", ev->desc); json_object_end(out); } diff --git a/plugins/bkpr/channel_event.h b/plugins/bkpr/channel_event.h index 28a078702..ec09858bb 100644 --- a/plugins/bkpr/channel_event.h +++ b/plugins/bkpr/channel_event.h @@ -43,6 +43,9 @@ struct channel_event { /* What time did the event happen */ u64 timestamp; + + /* Description, usually from invoice */ + const char *desc; }; struct channel_event *new_channel_event(const tal_t *ctx, diff --git a/plugins/bkpr/db.c b/plugins/bkpr/db.c index 75795c8f2..fdd9ebd2b 100644 --- a/plugins/bkpr/db.c +++ b/plugins/bkpr/db.c @@ -96,6 +96,8 @@ static struct migration db_migrations[] = { {SQL("ALTER TABLE accounts ADD closed_count INTEGER DEFAULT 0;"), NULL}, {SQL("ALTER TABLE chain_events ADD ignored INTEGER;"), NULL}, {SQL("ALTER TABLE chain_events ADD stealable INTEGER;"), NULL}, + {SQL("ALTER TABLE chain_events ADD ev_desc TEXT DEFAULT NULL;"), NULL}, + {SQL("ALTER TABLE channel_events ADD ev_desc TEXT DEFAULT NULL;"), NULL}, }; static bool db_migrate(struct plugin *p, struct db *db) diff --git a/plugins/bkpr/incomestmt.c b/plugins/bkpr/incomestmt.c index 0a2eab100..7e586d3de 100644 --- a/plugins/bkpr/incomestmt.c +++ b/plugins/bkpr/incomestmt.c @@ -47,16 +47,13 @@ static struct income_event *chain_to_income(const tal_t *ctx, inc->timestamp = ev->timestamp; inc->outpoint = tal_dup(inc, struct bitcoin_outpoint, &ev->outpoint); - if (ev->spending_txid) - inc->txid = tal_dup(inc, struct bitcoin_txid, - ev->spending_txid); + if (ev->desc) + inc->desc = tal_strdup(inc, ev->desc); else - inc->txid = NULL; + inc->desc = NULL; - if (ev->payment_id) - inc->payment_id = tal_dup(inc, struct sha256, ev->payment_id); - else - inc->payment_id = NULL; + inc->txid = tal_dup_or_null(inc, struct bitcoin_txid, ev->spending_txid); + inc->payment_id = tal_dup_or_null(inc, struct sha256, ev->payment_id); return inc; } @@ -76,10 +73,11 @@ static struct income_event *channel_to_income(const tal_t *ctx, inc->timestamp = ev->timestamp; inc->outpoint = NULL; inc->txid = NULL; - if (ev->payment_id) - inc->payment_id = tal_dup(inc, struct sha256, ev->payment_id); + if (ev->desc) + inc->desc = tal_strdup(inc, ev->desc); else - inc->payment_id = NULL; + inc->desc = NULL; + inc->payment_id = tal_dup_or_null(inc, struct sha256, ev->payment_id); return inc; } @@ -99,6 +97,7 @@ static struct income_event *onchainfee_to_income(const tal_t *ctx, inc->txid = tal_dup(inc, struct bitcoin_txid, &fee->txid); inc->outpoint = NULL; inc->payment_id = NULL; + inc->desc = NULL; return inc; } @@ -397,6 +396,9 @@ void json_add_income_event(struct json_stream *out, struct income_event *ev) json_add_string(out, "currency", ev->currency); json_add_u64(out, "timestamp", ev->timestamp); + if (ev->desc) + json_add_string(out, "description", ev->desc); + if (ev->outpoint) json_add_outpoint(out, "outpoint", ev->outpoint); @@ -570,7 +572,8 @@ static void koinly_entry(const tal_t *ctx, FILE *csvf, struct income_event *ev) fprintf(csvf, ","); /* Description */ - fprintf(csvf, "%s: account %s", ev->tag, ev->acct_name); + if (ev->desc) + fprintf(csvf, "%s", ev->desc); fprintf(csvf, ","); /* TxHash */ @@ -709,8 +712,8 @@ static void harmony_entry(const tal_t *ctx, FILE *csvf, struct income_event *ev) ev->outpoint)); fprintf(csvf, ","); - /* ",Note" account tag */ - fprintf(csvf, "%s %s", ev->acct_name, ev->tag); + /* ",Note" description (may be NULL) */ + fprintf(csvf, "%s", ev->desc ? ev->desc : ""); } static void quickbooks_header(FILE *csvf) @@ -733,12 +736,17 @@ static void quickbooks_entry(const tal_t *ctx, FILE *csvf, struct income_event * /* datefmt: dd/mm/yyyy */ char timebuf[sizeof("dd/mm/yyyy")]; strftime(timebuf, sizeof(timebuf), "%d/%m/%Y", gmtime(&tv)); + + /* New line! */ + fprintf(csvf, "\n"); + fprintf(csvf, "%s", timebuf); fprintf(csvf, ","); /* Description */ - fprintf(csvf, "%s (%s) in %s", - ev->tag, ev->acct_name, ev->currency); + fprintf(csvf, "%s (%s) %s: %s", + ev->tag, ev->acct_name, ev->currency, + ev->desc ? ev->desc : "no desc"); fprintf(csvf, ","); /* Credit */ diff --git a/plugins/bkpr/incomestmt.h b/plugins/bkpr/incomestmt.h index 3092e513b..12f288d59 100644 --- a/plugins/bkpr/incomestmt.h +++ b/plugins/bkpr/incomestmt.h @@ -8,6 +8,7 @@ struct income_event { char *acct_name; char *tag; + char *desc; struct amount_msat credit; struct amount_msat debit; char *currency; diff --git a/plugins/bkpr/recorder.c b/plugins/bkpr/recorder.c index 43f2ccc3a..9b1ab449f 100644 --- a/plugins/bkpr/recorder.c +++ b/plugins/bkpr/recorder.c @@ -58,6 +58,11 @@ static struct chain_event *stmt2chain_event(const tal_t *ctx, struct db_stmt *st e->ignored = db_col_int(stmt, "e.ignored") == 1; e->stealable = db_col_int(stmt, "e.stealable") == 1; + if (!db_col_is_null(stmt, "e.ev_desc")) + e->desc = db_col_strdup(e, stmt, "e.ev_desc"); + else + e->desc = NULL; + return e; } @@ -102,6 +107,11 @@ static struct channel_event *stmt2channel_event(const tal_t *ctx, struct db_stmt e->part_id = db_col_int(stmt, "e.part_id"); e->timestamp = db_col_u64(stmt, "e.timestamp"); + if (!db_col_is_null(stmt, "e.ev_desc")) + e->desc = db_col_strdup(e, stmt, "e.ev_desc"); + else + e->desc = NULL; + return e; } @@ -130,6 +140,7 @@ struct chain_event **list_chain_events_timebox(const tal_t *ctx, ", e.payment_id" ", e.ignored" ", e.stealable" + ", e.ev_desc" " FROM chain_events e" " LEFT OUTER JOIN accounts a" " ON e.account_id = a.id" @@ -171,6 +182,7 @@ struct chain_event **account_get_chain_events(const tal_t *ctx, ", e.payment_id" ", e.ignored" ", e.stealable" + ", e.ev_desc" " FROM chain_events e" " LEFT OUTER JOIN accounts a" " ON e.account_id = a.id" @@ -205,6 +217,7 @@ static struct chain_event **find_txos_for_tx(const tal_t *ctx, ", e.payment_id" ", e.ignored" ", e.stealable" + ", e.ev_desc" " FROM chain_events e" " LEFT OUTER JOIN accounts a" " ON e.account_id = a.id" @@ -586,6 +599,31 @@ void maybe_mark_account_onchain(struct db *db, struct account *acct) tal_free(ctx); } +void add_payment_hash_desc(struct db *db, + struct sha256 *payment_hash, + const char *desc) +{ + struct db_stmt *stmt; + + /* Ok, now we update the account with this blockheight */ + stmt = db_prepare_v2(db, SQL("UPDATE channel_events SET" + " ev_desc = ?" + " WHERE" + " payment_id = ?")); + db_bind_text(stmt, 0, desc); + db_bind_sha256(stmt, 1, payment_hash); + db_exec_prepared_v2(take(stmt)); + + /* Ok, now we update the account with this blockheight */ + stmt = db_prepare_v2(db, SQL("UPDATE chain_events SET" + " ev_desc = ?" + " WHERE" + " payment_id = ?")); + db_bind_text(stmt, 0, desc); + db_bind_sha256(stmt, 1, payment_hash); + db_exec_prepared_v2(take(stmt)); +} + struct chain_event *find_chain_event_by_id(const tal_t *ctx, struct db *db, u64 event_db_id) @@ -611,6 +649,7 @@ struct chain_event *find_chain_event_by_id(const tal_t *ctx, ", e.payment_id" ", e.ignored" ", e.stealable" + ", e.ev_desc" " FROM chain_events e" " LEFT OUTER JOIN accounts a" " ON e.account_id = a.id" @@ -657,6 +696,7 @@ static struct chain_event *find_chain_event(const tal_t *ctx, ", e.payment_id" ", e.ignored" ", e.stealable" + ", e.ev_desc" " FROM chain_events e" " LEFT OUTER JOIN accounts a" " ON e.account_id = a.id" @@ -685,6 +725,7 @@ static struct chain_event *find_chain_event(const tal_t *ctx, ", e.payment_id" ", e.ignored" ", e.stealable" + ", e.ev_desc" " FROM chain_events e" " LEFT OUTER JOIN accounts a" " ON e.account_id = a.id" @@ -836,6 +877,7 @@ struct channel_event **list_channel_events_timebox(const tal_t *ctx, ", e.payment_id" ", e.part_id" ", e.timestamp" + ", e.ev_desc" " FROM channel_events e" " LEFT OUTER JOIN accounts a" " ON a.id = e.account_id" @@ -882,6 +924,7 @@ struct channel_event **account_get_channel_events(const tal_t *ctx, ", e.payment_id" ", e.part_id" ", e.timestamp" + ", e.ev_desc" " FROM channel_events e" " LEFT OUTER JOIN accounts a" " ON a.id = e.account_id" @@ -1283,9 +1326,10 @@ void log_channel_event(struct db *db, ", payment_id" ", part_id" ", timestamp" + ", ev_desc" ")" " VALUES" - " (?, ?, ?, ?, ?, ?, ?, ?, ?);")); + " (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);")); db_bind_u64(stmt, 0, acct->db_id); db_bind_text(stmt, 1, e->tag); @@ -1299,6 +1343,10 @@ void log_channel_event(struct db *db, db_bind_null(stmt, 6); db_bind_int(stmt, 7, e->part_id); db_bind_u64(stmt, 8, e->timestamp); + if (e->desc) + db_bind_text(stmt, 9, e->desc); + else + db_bind_null(stmt, 9); db_exec_prepared_v2(stmt); e->db_id = db_last_insert_id_v2(stmt); @@ -1330,6 +1378,7 @@ static struct chain_event **find_chain_events_bytxid(const tal_t *ctx, struct db ", e.payment_id" ", e.ignored" ", e.stealable" + ", e.ev_desc" " FROM chain_events e" " LEFT OUTER JOIN accounts a" " ON a.id = e.account_id" @@ -1820,9 +1869,10 @@ bool log_chain_event(struct db *db, ", spending_txid" ", ignored" ", stealable" + ", ev_desc" ")" " VALUES " - "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);")); + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);")); db_bind_u64(stmt, 0, acct->db_id); if (e->origin_acct) @@ -1851,6 +1901,10 @@ bool log_chain_event(struct db *db, db_bind_int(stmt, 13, e->ignored ? 1 : 0); db_bind_int(stmt, 14, e->stealable ? 1 : 0); + if (e->desc) + db_bind_text(stmt, 15, e->desc); + else + db_bind_null(stmt, 15); db_exec_prepared_v2(stmt); e->db_id = db_last_insert_id_v2(stmt); e->acct_db_id = acct->db_id; diff --git a/plugins/bkpr/recorder.h b/plugins/bkpr/recorder.h index c7c4209ca..83714c05e 100644 --- a/plugins/bkpr/recorder.h +++ b/plugins/bkpr/recorder.h @@ -184,6 +184,14 @@ char *update_channel_onchain_fees(const tal_t *ctx, * The point of this is to allow us to prune data, eventually */ void maybe_mark_account_onchain(struct db *db, struct account *acct); +/* We fetch invoice desc data after the fact and then update it + * Updates both the chain_event and channel_event tables for all + * matching payment_hashes + * */ +void add_payment_hash_desc(struct db *db, + struct sha256 *payment_hash, + const char *desc); + /* When we make external deposits from the wallet, we don't * count them until any output that was spent *into* them is * confirmed onchain. diff --git a/plugins/bkpr/test/run-recorder.c b/plugins/bkpr/test/run-recorder.c index 6af1ea6a6..b62cedb61 100644 --- a/plugins/bkpr/test/run-recorder.c +++ b/plugins/bkpr/test/run-recorder.c @@ -287,6 +287,10 @@ static bool channel_events_eq(struct channel_event *e1, struct channel_event *e2 CHECK(e1->part_id == e2->part_id); CHECK(e1->timestamp == e2->timestamp); + CHECK((e1->desc != NULL) == (e2->desc != NULL)); + if (e1->desc) + CHECK(streq(e1->desc, e2->desc)); + return true; } @@ -314,6 +318,9 @@ static bool chain_events_eq(struct chain_event *e1, struct chain_event *e2) if (e1->payment_id) CHECK(sha256_eq(e1->payment_id, e2->payment_id)); + CHECK((e1->desc != NULL) == (e2->desc != NULL)); + if (e1->desc) + CHECK(streq(e1->desc, e2->desc)); return true; } @@ -335,6 +342,7 @@ static struct channel_event *make_channel_event(const tal_t *ctx, ev->timestamp = 1919191; ev->part_id = 19; ev->tag = tag; + ev->desc = tal_fmt(ev, "description"); return ev; } @@ -364,6 +372,7 @@ static struct chain_event *make_chain_event(const tal_t *ctx, ev->blockheight = blockheight; ev->ignored = false; ev->stealable = false; + ev->desc = tal_fmt(ev, "hello hello"); memset(&ev->outpoint.txid, outpoint_char, sizeof(struct bitcoin_txid)); ev->outpoint.n = outnum; @@ -880,9 +889,11 @@ static bool test_channel_event_crud(const tal_t *ctx, struct plugin *p) ev1->currency = "btc"; ev1->timestamp = 11111; ev1->part_id = 19; + ev1->desc = tal_strdup(ev1, "hello desc1"); /* Passing unknown tags in should be ok */ ev1->tag = "hello"; + ev1->desc = tal_fmt(ev1, "desc"); ev2 = tal(ctx, struct channel_event); ev2->payment_id = tal(ev2, struct sha256); @@ -894,6 +905,7 @@ static bool test_channel_event_crud(const tal_t *ctx, struct plugin *p) ev2->timestamp = 22222; ev2->part_id = 0; ev2->tag = tal_fmt(ev2, "deposit"); + ev2->desc = NULL; ev3 = tal(ctx, struct channel_event); ev3->payment_id = tal(ev3, struct sha256); @@ -905,6 +917,7 @@ static bool test_channel_event_crud(const tal_t *ctx, struct plugin *p) ev3->timestamp = 33333; ev3->part_id = 5; ev3->tag = tal_fmt(ev3, "routed"); + ev3->desc = NULL; db_begin_transaction(db); log_channel_event(db, acct, ev1); @@ -972,6 +985,7 @@ static bool test_chain_event_crud(const tal_t *ctx, struct plugin *p) ev1->spending_txid = tal(ctx, struct bitcoin_txid); memset(ev1->spending_txid, 'C', sizeof(struct bitcoin_txid)); ev1->payment_id = NULL; + ev1->desc = tal_fmt(ev1, "description"); db_begin_transaction(db); log_chain_event(db, acct, ev1); @@ -992,6 +1006,7 @@ static bool test_chain_event_crud(const tal_t *ctx, struct plugin *p) ev2->outpoint.n = 1; ev2->spending_txid = NULL; ev2->payment_id = tal(ctx, struct sha256); + ev2->desc = NULL; memset(ev2->payment_id, 'B', sizeof(struct sha256)); /* Dummy event, logged to separate account */ @@ -1011,6 +1026,7 @@ static bool test_chain_event_crud(const tal_t *ctx, struct plugin *p) ev3->spending_txid = tal(ctx, struct bitcoin_txid); memset(ev3->spending_txid, 'D', sizeof(struct bitcoin_txid)); ev3->payment_id = NULL; + ev3->desc = NULL; db_begin_transaction(db); log_chain_event(db, acct, ev2); @@ -1238,6 +1254,7 @@ static bool test_account_crud(const tal_t *ctx, struct plugin *p) ev1->spending_txid = tal(ctx, struct bitcoin_txid); memset(ev1->spending_txid, 'C', sizeof(struct bitcoin_txid)); ev1->payment_id = NULL; + ev1->desc = tal_fmt(ev1, "oh hello"); db_begin_transaction(db); log_chain_event(db, acct, ev1); diff --git a/tests/test_bookkeeper.py b/tests/test_bookkeeper.py index b082504e5..1cd013d8f 100644 --- a/tests/test_bookkeeper.py +++ b/tests/test_bookkeeper.py @@ -376,3 +376,45 @@ def test_bookkeeping_onchaind_txs(node_factory, bitcoind): assert len(funds['channels']) == 0 outs = sum([out['amount_msat'] for out in funds['outputs']]) assert outs == only_one(wallet_bal['balances'])['balance_msat'] + + +def test_bookkeeping_descriptions(node_factory, bitcoind, chainparams): + """ + When an 'invoice' type event comes through, we look up the description details + to include about the item. Particularly useful for CSV outputs etc. + """ + l1, l2 = node_factory.line_graph(2, opts={'experimental-offers': None}) + + # Send l2 funds via the channel + bolt11_desc = "test bolt11 description" + l1.pay(l2, 11000000, label=bolt11_desc) + l1.daemon.wait_for_log('coin_move .* [(]invoice[)] 0msat -11000000msat') + l2.daemon.wait_for_log('coin_move .* [(]invoice[)] 11000000msat') + + # Test paying an bolt11 invoice (rcvr) + l1_inc_ev = l1.rpc.bkpr_listincome()['income_events'] + inv = only_one([ev for ev in l1_inc_ev if ev['tag'] == 'invoice']) + assert inv['description'] == bolt11_desc + + # Test paying an bolt11 invoice (sender) + l2_inc_ev = l2.rpc.bkpr_listincome()['income_events'] + inv = only_one([ev for ev in l2_inc_ev if ev['tag'] == 'invoice']) + assert inv['description'] == bolt11_desc + + # Make an offer (l1) + bolt12_desc = "test bolt12 description" + offer = l1.rpc.call('offer', [100, bolt12_desc]) + invoice = l2.rpc.call('fetchinvoice', {'offer': offer['bolt12']}) + paid = l2.rpc.pay(invoice['invoice']) + l1.daemon.wait_for_log('coin_move .* [(]invoice[)] 100msat') + l2.daemon.wait_for_log('coin_move .* [(]invoice[)] 0msat -100msat') + + # Test paying an offer (bolt12) (rcvr) + l1_inc_ev = l1.rpc.bkpr_listincome()['income_events'] + inv = only_one([ev for ev in l1_inc_ev if 'payment_id' in ev and ev['payment_id'] == paid['payment_hash']]) + assert inv['description'] == bolt12_desc + + # Test paying an offer (bolt12) (sender) + l2_inc_ev = l2.rpc.bkpr_listincome()['income_events'] + inv = only_one([ev for ev in l2_inc_ev if 'payment_id' in ev and ev['payment_id'] == paid['payment_hash'] and ev['tag'] == 'invoice']) + assert inv['description'] == bolt12_desc diff --git a/tests/test_invoices.py b/tests/test_invoices.py index ccce09218..9cfd63ade 100644 --- a/tests/test_invoices.py +++ b/tests/test_invoices.py @@ -737,3 +737,8 @@ def test_invoice_deschash(node_factory, chainparams): with pytest.raises(RpcError, match=r'description already removed'): l2.rpc.delinvoice('label', "paid", desconly=True) + + # desc-hashes lands in bookkeeper data (description) + wait_for(lambda: len([ev for ev in l1.rpc.bkpr_listincome()['income_events'] if ev['tag'] == 'invoice']) == 1) + inv = only_one([ev for ev in l1.rpc.bkpr_listincome()['income_events'] if ev['tag'] == 'invoice']) + assert inv['description'] == b11['description_hash']