bkpr: save invoice description data to the database and display it

It'll be really nice to be able to read description data about an
invoice, if we've got it!

doc/lightning-bkpr-listaccountevents.7.md

@@ -38,6 +38,7 @@ If **type** is "chain":
   - **origin** (string, optional): The account this movement originated from
   - **payment_id** (hex, optional): lightning payment identifier. For an htlc, this will be the preimage.
   - **txid** (txid, optional): The txid of the transaction that created this event
+  - **description** (string, optional): The description of this event
 
 If **type** is "onchain_fee":
   - **txid** (txid): The txid of the transaction that created this event
@@ -65,4 +66,4 @@ RESOURCES
 
 Main web site: <https://github.com/ElementsProject/lightning>
 
-[comment]: # ( SHA256STAMP:dd72cc73e685daa6877984be8edede76dfec2f9d85df9a88ab1b031a93b20549)
+[comment]: # ( SHA256STAMP:f8538b1d1e6cda7cd801690e5c09741c8a843b27cc922065598914516c16d2b3)

doc/lightning-bkpr-listincome.7.md

@@ -33,6 +33,7 @@ On success, an object containing **income_events** is returned.  It is an array
 - **debit_msat** (msat): Amount spent (expenses)
 - **currency** (string): human-readable bech32 part for this coin type
 - **timestamp** (u32): Timestamp this event was recorded by the node. For consolidated events such as onchain_fees, the most recent timestamp
+- **description** (string, optional): More information about this event. If a `invoice` type, typically the bolt11/bolt12 description
 - **outpoint** (string, optional): The txid:outnum for this event, if applicable
 - **txid** (txid, optional): The txid of the transaction that created this event, if applicable
 - **payment_id** (hex, optional): lightning payment identifier. For an htlc, this will be the preimage.
@@ -55,4 +56,4 @@ RESOURCES
 
 Main web site: <https://github.com/ElementsProject/lightning>
 
-[comment]: # ( SHA256STAMP:400ac5e6719a7ae5ec7078a2cd220d91ab7e66ad45f08b46257e6ec04dcdeb4c)
+[comment]: # ( SHA256STAMP:ab8508af0f40587c5a804f6981591564fe2d18b4fe3fbe7793e6a489607f7e0a)

doc/schemas/bkpr-listaccountevents.schema.json

@@ -95,6 +95,10 @@
                 "txid": {
                   "type": "txid",
                   "description": "The txid of the transaction that created this event"
+                },
+                "description": {
+                  "type": "string",
+                  "description": "The description of this event"
                 }
               },
               "required": [
@@ -124,6 +128,7 @@
                 "debit_msat": {},
                 "currency": {},
                 "timestamp": {},
+                "description": {},
                 "txid": {
                   "type": "txid",
                   "description": "The txid of the transaction that created this event"
@@ -155,6 +160,7 @@
                 "debit_msat": {},
                 "currency": {},
                 "timestamp": {},
+                "description": {},
                 "fees_msat": {
                   "type": "msat",
                   "description": "Amount paid in fees"

doc/schemas/bkpr-listincome.schema.json

@@ -44,6 +44,10 @@
             "type": "u32",
             "description": "Timestamp this event was recorded by the node. For consolidated events such as onchain_fees, the most recent timestamp"
           },
+          "description": {
+            "type": "string",
+            "description": "More information about this event. If a `invoice` type, typically the bolt11/bolt12 description"
+          },
           "outpoint": {
             "type": "string",
             "description": "The txid:outnum for this event, if applicable"

plugins/bkpr/Makefile

@@ -37,7 +37,7 @@ PLUGIN_ALL_HEADER += $(BOOKKEEPER_HEADER)
 C_PLUGINS += plugins/bookkeeper
 PLUGINS += plugins/bookkeeper
 
-plugins/bookkeeper: bitcoin/chainparams.o common/coin_mvt.o $(BOOKKEEPER_OBJS) $(PLUGIN_LIB_OBJS) $(JSMN_OBJTS) $(PLUGIN_COMMON_OBJS) $(WIRE_OBJS) $(DB_OBJS)
+plugins/bookkeeper: bitcoin/chainparams.o common/bolt12.o common/bolt12_merkle.o $(BOOKKEEPER_OBJS) $(PLUGIN_LIB_OBJS) $(JSMN_OBJTS) $(PLUGIN_COMMON_OBJS) $(WIRE_OBJS) $(DB_OBJS)
 
 # The following files contain SQL-annotated statements that we need to extact
 BOOKKEEPER_SQL_FILES := 			\

plugins/bkpr/bookkeeper.c

@@ -2,7 +2,10 @@
 #include <ccan/array_size/array_size.h>
 #include <ccan/cast/cast.h>
 #include <ccan/tal/str/str.h>
+#include <ccan/tal/tal.h>
 #include <ccan/time/time.h>
+#include <common/bolt11.h>
+#include <common/bolt12.h>
 #include <common/coin_mvt.h>
 #include <common/json_param.h>
 #include <common/json_stream.h>
@@ -623,6 +626,7 @@ static bool new_missed_channel_account(struct command *cmd,
 			chain_ev->payment_id = NULL;
 			chain_ev->ignored = false;
 			chain_ev->stealable = false;
+			chain_ev->desc = NULL;
 
 			/* Update the account info too */
 			tags = tal_arr(chain_ev, enum mvt_tag, 1);
@@ -798,7 +802,7 @@ static struct command_result *log_error(struct command *cmd,
 					void *arg UNNEEDED)
 {
 	plugin_log(cmd->plugin, LOG_BROKEN,
-		   "error calling `listpeers`: %.*s",
+		   "error calling rpc: %.*s",
 		   json_tok_full_len(error),
 		   json_tok_full(buf, error));
 
@@ -898,70 +902,6 @@ static char *do_account_close_checks(const tal_t *ctx,
 	return NULL;
 }
 
-struct event_info {
-	struct chain_event *ev;
-	struct account *acct;
-};
-
-static struct command_result *
-listpeers_done(struct command *cmd, const char *buf,
-	       const jsmntok_t *result, struct event_info *info)
-{
-	struct acct_balance **balances, *bal;
-	struct amount_msat credit_diff, debit_diff;
-	const char *err;
-	/* Make sure to clean up when we're done */
-	tal_steal(cmd, info);
-
-	if (new_missed_channel_account(cmd, buf, result,
-					info->acct,
-					info->ev->currency,
-					info->ev->timestamp)) {
-		db_begin_transaction(db);
-		err = account_get_balance(tmpctx, db, info->acct->name,
-					  false, false, &balances);
-		db_commit_transaction(db);
-
-		if (err)
-			plugin_err(cmd->plugin, err);
-
-		/* FIXME: multiple currencies per account? */
-		if (tal_count(balances) > 0)
-			bal = balances[0];
-		else {
-			bal = tal(balances, struct acct_balance);
-			bal->credit = AMOUNT_MSAT(0);
-			bal->debit = AMOUNT_MSAT(0);
-		}
-		assert(tal_count(balances) == 1);
-
-		/* The expected current balance is zero, since
-		 * we just got the channel close event */
-		err = msat_find_diff(AMOUNT_MSAT(0),
-				     bal->credit,
-				     bal->debit,
-				     &credit_diff, &debit_diff);
-		if (err)
-			plugin_err(cmd->plugin, err);
-
-		log_journal_entry(info->acct,
-				  info->ev->currency,
-				  info->ev->timestamp - 1,
-				  credit_diff, debit_diff);
-	} else
-		plugin_log(cmd->plugin, LOG_BROKEN,
-			   "Unable to find account %s in listpeers",
-			   info->acct->name);
-
-	/* Maybe mark acct as onchain resolved */
-	err = do_account_close_checks(cmd, info->ev, info->acct);
-	if (err)
-		plugin_err(cmd->plugin, err);
-
-	return notification_handled(cmd);
-}
-
-
 static struct command_result *json_balance_snapshot(struct command *cmd,
 						    const char *buf,
 						    const jsmntok_t *params)
@@ -1144,6 +1084,242 @@ static struct command_result *json_balance_snapshot(struct command *cmd,
 	return notification_handled(cmd);
 }
 
+/* Returns true if "fatal" error, otherwise just a normal error */
+static char *fetch_out_desc_invstr(const tal_t *ctx, const char *buf,
+				   const jsmntok_t *tok, char **err)
+{
+	char *bolt, *desc, *fail;
+
+	/* It's a bolt11! Parse it out to a desc */
+	if (!json_scan(ctx, buf, tok, "{bolt11:%}",
+		       JSON_SCAN_TAL(ctx, json_strdup, &bolt))) {
+		struct bolt11 *bolt11;
+		u5 *sigdata;
+		struct sha256 hash;
+		bool have_n;
+
+		bolt11 = bolt11_decode_nosig(ctx, bolt,
+				       /* No desc/features/chain checks */
+				       NULL, NULL, NULL,
+				       &hash, &sigdata, &have_n,
+				       &fail);
+
+		if (bolt11) {
+			if (bolt11->description)
+				desc = tal_strdup(ctx, bolt11->description);
+			else if (bolt11->description_hash)
+				desc = tal_fmt(ctx, "%s",
+					       type_to_string(ctx,
+						      struct sha256,
+						      bolt11->description_hash));
+			else
+				desc = NULL;
+		} else {
+			*err = tal_fmt(ctx, "failed to parse bolt11 %s: %s",
+				       bolt, fail);
+			return NULL;
+		}
+	} else if (!json_scan(ctx, buf, tok, "{bolt12:%}",
+			      JSON_SCAN_TAL(ctx, json_strdup, &bolt))) {
+		struct tlv_invoice *bolt12;
+
+		bolt12 = invoice_decode_nosig(ctx, bolt, strlen(bolt),
+					      /* No features/chain checks */
+					      NULL, NULL,
+					      &fail);
+		if (!bolt12) {
+			*err = tal_fmt(ctx, "failed to parse"
+				       " bolt12 %s: %s",
+				       bolt, fail);
+			return NULL;
+		}
+
+		if (bolt12->description)
+			desc = tal_strndup(ctx,
+				cast_signed(char *, bolt12->description),
+				tal_bytelen(bolt12->description));
+		else
+			desc = NULL;
+	} else
+		desc = NULL;
+
+	*err = NULL;
+	return desc;
+}
+
+static struct command_result *
+listinvoice_done(struct command *cmd, const char *buf,
+		 const jsmntok_t *result, struct sha256 *payment_hash)
+{
+	size_t i;
+	const jsmntok_t *inv_arr_tok, *inv_tok;
+	const char *desc;
+	inv_arr_tok = json_get_member(buf, result, "invoices");
+	assert(inv_arr_tok->type == JSMN_ARRAY);
+
+	desc = NULL;
+	json_for_each_arr(i, inv_tok, inv_arr_tok) {
+		char *err;
+
+		/* Found desc in "description" */
+		if (!json_scan(cmd, buf, inv_tok, "{description:%}",
+				JSON_SCAN_TAL(cmd, json_strdup, &desc)))
+			break;
+
+		/* if 'description' doesn't exist, try bolt11/bolt12 */
+		desc = fetch_out_desc_invstr(cmd, buf, inv_tok, &err);
+		if (desc || err) {
+			if (err)
+				plugin_log(cmd->plugin,
+					   LOG_BROKEN, "%s", err);
+			break;
+		}
+	}
+
+	if (desc) {
+		db_begin_transaction(db);
+		add_payment_hash_desc(db, payment_hash, desc);
+		db_commit_transaction(db);
+	} else
+		plugin_log(cmd->plugin, LOG_DBG,
+			   "listinvoices:"
+			   " description/bolt11/bolt12"
+			   " not found (%.*s)",
+			   result->end - result->start, buf);
+
+	return notification_handled(cmd);
+}
+
+static struct command_result *
+listsendpays_done(struct command *cmd, const char *buf,
+		  const jsmntok_t *result, struct sha256 *payment_hash)
+{
+	size_t i;
+	const jsmntok_t *pays_arr_tok, *pays_tok;
+	const char *desc;
+	pays_arr_tok = json_get_member(buf, result, "payments");
+	assert(pays_arr_tok->type == JSMN_ARRAY);
+
+	/* Did we find a matching entry? */
+	desc = NULL;
+	json_for_each_arr(i, pays_tok, pays_arr_tok) {
+		char *err;
+
+		desc = fetch_out_desc_invstr(cmd, buf, pays_tok, &err);
+		if (desc || err) {
+			if (err)
+				plugin_log(cmd->plugin,
+					   LOG_BROKEN, "%s", err);
+			break;
+		}
+	}
+
+	if (desc) {
+		db_begin_transaction(db);
+		add_payment_hash_desc(db, payment_hash, desc);
+		db_commit_transaction(db);
+	} else
+		plugin_log(cmd->plugin, LOG_DBG,
+			   "listpays: bolt11/bolt12 not found:"
+			   "(%.*s)",
+			   result->end - result->start, buf);
+
+	return notification_handled(cmd);
+}
+
+static struct command_result *lookup_invoice_desc(struct command *cmd,
+						  struct amount_msat credit,
+						  struct amount_msat debit,
+						  struct sha256 *payment_hash)
+{
+	struct out_req *req;
+
+	if (!amount_msat_zero(credit))
+		req = jsonrpc_request_start(cmd->plugin, cmd,
+					    "listinvoices",
+					    listinvoice_done,
+					    log_error,
+					    payment_hash);
+	else
+		req = jsonrpc_request_start(cmd->plugin, cmd,
+					    "listsendpays",
+					    listsendpays_done,
+					    log_error,
+					    payment_hash);
+
+	json_add_sha256(req->js, "payment_hash", payment_hash);
+	send_outreq(cmd->plugin, req);
+	return command_still_pending(cmd);
+}
+
+struct event_info {
+	struct chain_event *ev;
+	struct account *acct;
+};
+
+static struct command_result *
+listpeers_done(struct command *cmd, const char *buf,
+	       const jsmntok_t *result, struct event_info *info)
+{
+	struct acct_balance **balances, *bal;
+	struct amount_msat credit_diff, debit_diff;
+	const char *err;
+	/* Make sure to clean up when we're done */
+	tal_steal(cmd, info);
+
+	if (new_missed_channel_account(cmd, buf, result,
+					info->acct,
+					info->ev->currency,
+					info->ev->timestamp)) {
+		db_begin_transaction(db);
+		err = account_get_balance(tmpctx, db, info->acct->name,
+					  false, false, &balances);
+		db_commit_transaction(db);
+
+		if (err)
+			plugin_err(cmd->plugin, err);
+
+		/* FIXME: multiple currencies per account? */
+		if (tal_count(balances) > 0)
+			bal = balances[0];
+		else {
+			bal = tal(balances, struct acct_balance);
+			bal->credit = AMOUNT_MSAT(0);
+			bal->debit = AMOUNT_MSAT(0);
+		}
+		assert(tal_count(balances) == 1);
+
+		/* The expected current balance is zero, since
+		 * we just got the channel close event */
+		err = msat_find_diff(AMOUNT_MSAT(0),
+				     bal->credit,
+				     bal->debit,
+				     &credit_diff, &debit_diff);
+		if (err)
+			plugin_err(cmd->plugin, err);
+
+		log_journal_entry(info->acct,
+				  info->ev->currency,
+				  info->ev->timestamp - 1,
+				  credit_diff, debit_diff);
+	} else
+		plugin_log(cmd->plugin, LOG_BROKEN,
+			   "Unable to find account %s in listpeers",
+			   info->acct->name);
+
+	/* Maybe mark acct as onchain resolved */
+	err = do_account_close_checks(cmd, info->ev, info->acct);
+	if (err)
+		plugin_err(cmd->plugin, err);
+
+	if (info->ev->payment_id &&
+	    streq(info->ev->tag, mvt_tag_str(INVOICE)))
+		return lookup_invoice_desc(cmd, info->ev->credit,
+					   info->ev->debit,
+					   info->ev->payment_id);
+
+	return notification_handled(cmd);
+}
 static struct command_result *
 parse_and_log_chain_move(struct command *cmd,
 			 const char *buf,
@@ -1153,7 +1329,8 @@ parse_and_log_chain_move(struct command *cmd,
 			 const struct amount_msat debit,
 			 const char *coin_type STEALS,
 			 const u64 timestamp,
-			 const enum mvt_tag *tags)
+			 const enum mvt_tag *tags,
+			 const char *desc)
 {
 	struct chain_event *e = tal(cmd, struct chain_event);
 	struct sha256 *payment_hash = tal(cmd, struct sha256);
@@ -1247,6 +1424,7 @@ parse_and_log_chain_move(struct command *cmd,
 	e->currency = tal_steal(e, coin_type);
 	e->timestamp = timestamp;
 	e->tag = mvt_tag_str(tags[0]);
+	e->desc = tal_steal(e, desc);
 
 	e->ignored = false;
 	e->stealable = false;
@@ -1255,7 +1433,6 @@ parse_and_log_chain_move(struct command *cmd,
 		e->stealable |= tags[i] == STEALABLE;
 	}
 
-
 	db_begin_transaction(db);
 	acct = find_account(cmd, db, acct_name);
 
@@ -1349,6 +1526,18 @@ parse_and_log_chain_move(struct command *cmd,
 	if (err)
 		plugin_err(cmd->plugin, err);
 
+	/* Check for invoice desc data, necessary */
+	if (e->payment_id) {
+		for (size_t i = 0; i < tal_count(tags); i++) {
+			if (tags[i] != INVOICE)
+				continue;
+
+			return lookup_invoice_desc(cmd, e->credit,
+						   e->debit,
+						   e->payment_id);
+		}
+	}
+
 	return notification_handled(cmd);;
 }
 
@@ -1361,7 +1550,8 @@ parse_and_log_channel_move(struct command *cmd,
 			   const struct amount_msat debit,
 			   const char *coin_type STEALS,
 			   const u64 timestamp,
-			   const enum mvt_tag *tags)
+			   const enum mvt_tag *tags,
+			   const char *desc)
 {
 	struct channel_event *e = tal(cmd, struct channel_event);
 	struct account *acct;
@@ -1397,6 +1587,7 @@ parse_and_log_channel_move(struct command *cmd,
 	e->currency = tal_steal(e, coin_type);
 	e->timestamp = timestamp;
 	e->tag = mvt_tag_str(tags[0]);
+	e->desc = tal_steal(e, desc);
 
 	/* Go find the account for this event */
 	db_begin_transaction(db);
@@ -1410,6 +1601,18 @@ parse_and_log_channel_move(struct command *cmd,
 	log_channel_event(db, acct, e);
 	db_commit_transaction(db);
 
+	/* Check for invoice desc data, necessary */
+	if (e->payment_id) {
+		for (size_t i = 0; i < tal_count(tags); i++) {
+			if (tags[i] != INVOICE)
+				continue;
+
+			return lookup_invoice_desc(cmd, e->credit,
+						   e->debit,
+						   e->payment_id);
+		}
+	}
+
 	return notification_handled(cmd);
 }
 
@@ -1434,9 +1637,9 @@ static char *parse_tags(const tal_t *ctx,
 	return NULL;
 }
 
-static struct command_result * json_coin_moved(struct command *cmd,
+static struct command_result *json_coin_moved(struct command *cmd,
-					       const char *buf,
+					      const char *buf,
-					       const jsmntok_t *params)
+					      const jsmntok_t *params)
 {
 	const char *err, *mvt_type, *acct_name, *coin_type;
 	u32 version;
@@ -1490,13 +1693,15 @@ static struct command_result * json_coin_moved(struct command *cmd,
 	if (streq(mvt_type, CHAIN_MOVE))
 		return parse_and_log_chain_move(cmd, buf, params,
 					        acct_name, credit, debit,
-					        coin_type, timestamp, tags);
+					        coin_type, timestamp, tags,
+						NULL);
 
 
 	assert(streq(mvt_type, CHANNEL_MOVE));
 	return parse_and_log_channel_move(cmd, buf, params,
 					  acct_name, credit, debit,
-					  coin_type, timestamp, tags);
+					  coin_type, timestamp, tags,
+					  NULL);
 }
 
 const struct plugin_notification notifs[] = {

plugins/bkpr/chain_event.c

@@ -22,5 +22,7 @@ void json_add_chain_event(struct json_stream *out, struct chain_event *ev)
 		json_add_sha256(out, "payment_id", ev->payment_id);
 	json_add_u64(out, "timestamp", ev->timestamp);
 	json_add_u32(out, "blockheight", ev->blockheight);
+	if (ev->desc)
+		json_add_string(out, "description", ev->desc);
 	json_object_end(out);
 }

plugins/bkpr/chain_event.h

@@ -60,6 +60,9 @@ struct chain_event {
 
 	/* Sometimes chain events resolve payments */
 	struct sha256 *payment_id;
+
+	/* Desc of event (maybe useful for printing notes) */
+	const char *desc;
 };
 
 void json_add_chain_event(struct json_stream *out,

plugins/bkpr/channel_event.c

@@ -26,6 +26,7 @@ struct channel_event *new_channel_event(const tal_t *ctx,
 	ev->payment_id = tal_steal(ev, payment_id);
 	ev->part_id = part_id;
 	ev->timestamp = timestamp;
+	ev->desc = NULL;
 
 	return ev;
 }
@@ -47,5 +48,7 @@ void json_add_channel_event(struct json_stream *out,
 		json_add_u32(out, "part_id", ev->part_id);
 	}
 	json_add_u64(out, "timestamp", ev->timestamp);
+	if (ev->desc)
+		json_add_string(out, "description", ev->desc);
 	json_object_end(out);
 }

plugins/bkpr/channel_event.h

@@ -43,6 +43,9 @@ struct channel_event {
 
 	/* What time did the event happen */
 	u64 timestamp;
+
+	/* Description, usually from invoice */
+	const char *desc;
 };
 
 struct channel_event *new_channel_event(const tal_t *ctx,

plugins/bkpr/db.c

@@ -96,6 +96,8 @@ static struct migration db_migrations[] = {
 	{SQL("ALTER TABLE accounts ADD closed_count INTEGER DEFAULT 0;"), NULL},
 	{SQL("ALTER TABLE chain_events ADD ignored INTEGER;"), NULL},
 	{SQL("ALTER TABLE chain_events ADD stealable INTEGER;"), NULL},
+	{SQL("ALTER TABLE chain_events ADD ev_desc TEXT DEFAULT NULL;"), NULL},
+	{SQL("ALTER TABLE channel_events ADD ev_desc TEXT DEFAULT NULL;"), NULL},
 };
 
 static bool db_migrate(struct plugin *p, struct db *db)

plugins/bkpr/incomestmt.c

@@ -47,16 +47,13 @@ static struct income_event *chain_to_income(const tal_t *ctx,
 	inc->timestamp = ev->timestamp;
 	inc->outpoint = tal_dup(inc, struct bitcoin_outpoint, &ev->outpoint);
 
-	if (ev->spending_txid)
+	if (ev->desc)
-		inc->txid = tal_dup(inc, struct bitcoin_txid,
+		inc->desc = tal_strdup(inc, ev->desc);
-				    ev->spending_txid);
 	else
-		inc->txid = NULL;
+		inc->desc = NULL;
 
-	if (ev->payment_id)
+	inc->txid = tal_dup_or_null(inc, struct bitcoin_txid, ev->spending_txid);
-		inc->payment_id = tal_dup(inc, struct sha256, ev->payment_id);
+	inc->payment_id = tal_dup_or_null(inc, struct sha256, ev->payment_id);
-	else
-		inc->payment_id = NULL;
 
 	return inc;
 }
@@ -76,10 +73,11 @@ static struct income_event *channel_to_income(const tal_t *ctx,
 	inc->timestamp = ev->timestamp;
 	inc->outpoint = NULL;
 	inc->txid = NULL;
-	if (ev->payment_id)
+	if (ev->desc)
-		inc->payment_id = tal_dup(inc, struct sha256, ev->payment_id);
+		inc->desc = tal_strdup(inc, ev->desc);
 	else
-		inc->payment_id = NULL;
+		inc->desc = NULL;
+	inc->payment_id = tal_dup_or_null(inc, struct sha256, ev->payment_id);
 
 	return inc;
 }
@@ -99,6 +97,7 @@ static struct income_event *onchainfee_to_income(const tal_t *ctx,
 	inc->txid = tal_dup(inc, struct bitcoin_txid, &fee->txid);
 	inc->outpoint = NULL;
 	inc->payment_id = NULL;
+	inc->desc = NULL;
 
 	return inc;
 }
@@ -397,6 +396,9 @@ void json_add_income_event(struct json_stream *out, struct income_event *ev)
 	json_add_string(out, "currency", ev->currency);
 	json_add_u64(out, "timestamp", ev->timestamp);
 
+	if (ev->desc)
+		json_add_string(out, "description", ev->desc);
+
 	if (ev->outpoint)
 		json_add_outpoint(out, "outpoint", ev->outpoint);
 
@@ -570,7 +572,8 @@ static void koinly_entry(const tal_t *ctx, FILE *csvf, struct income_event *ev)
 	fprintf(csvf, ",");
 
 	/* Description */
-	fprintf(csvf, "%s: account %s", ev->tag, ev->acct_name);
+	if (ev->desc)
+		fprintf(csvf, "%s", ev->desc);
 	fprintf(csvf, ",");
 
 	/* TxHash */
@@ -709,8 +712,8 @@ static void harmony_entry(const tal_t *ctx, FILE *csvf, struct income_event *ev)
 				       ev->outpoint));
 	fprintf(csvf, ",");
 
-	/* ",Note"  account tag */
+	/* ",Note"  description (may be NULL) */
-	fprintf(csvf, "%s %s", ev->acct_name, ev->tag);
+	fprintf(csvf, "%s", ev->desc ? ev->desc : "");
 }
 
 static void quickbooks_header(FILE *csvf)
@@ -733,12 +736,17 @@ static void quickbooks_entry(const tal_t *ctx, FILE *csvf, struct income_event *
 	/* datefmt: dd/mm/yyyy */
 	char timebuf[sizeof("dd/mm/yyyy")];
 	strftime(timebuf, sizeof(timebuf), "%d/%m/%Y", gmtime(&tv));
+
+	/* New line! */
+	fprintf(csvf, "\n");
+
 	fprintf(csvf, "%s", timebuf);
 	fprintf(csvf, ",");
 
 	/* Description */
-	fprintf(csvf, "%s (%s) in %s",
+	fprintf(csvf, "%s (%s) %s: %s",
-		ev->tag, ev->acct_name, ev->currency);
+		ev->tag, ev->acct_name, ev->currency,
+		ev->desc ? ev->desc : "no desc");
 	fprintf(csvf, ",");
 
 	/* Credit */

plugins/bkpr/incomestmt.h

@@ -8,6 +8,7 @@
 struct income_event {
 	char *acct_name;
 	char *tag;
+	char *desc;
 	struct amount_msat credit;
 	struct amount_msat debit;
 	char *currency;

plugins/bkpr/recorder.c

@@ -58,6 +58,11 @@ static struct chain_event *stmt2chain_event(const tal_t *ctx, struct db_stmt *st
 	e->ignored = db_col_int(stmt, "e.ignored") == 1;
 	e->stealable = db_col_int(stmt, "e.stealable") == 1;
 
+	if (!db_col_is_null(stmt, "e.ev_desc"))
+		e->desc = db_col_strdup(e, stmt, "e.ev_desc");
+	else
+		e->desc = NULL;
+
 	return e;
 }
 
@@ -102,6 +107,11 @@ static struct channel_event *stmt2channel_event(const tal_t *ctx, struct db_stmt
 	e->part_id = db_col_int(stmt, "e.part_id");
 	e->timestamp = db_col_u64(stmt, "e.timestamp");
 
+	if (!db_col_is_null(stmt, "e.ev_desc"))
+		e->desc = db_col_strdup(e, stmt, "e.ev_desc");
+	else
+		e->desc = NULL;
+
 	return e;
 }
 
@@ -130,6 +140,7 @@ struct chain_event **list_chain_events_timebox(const tal_t *ctx,
 				     ", e.payment_id"
 				     ", e.ignored"
 				     ", e.stealable"
+				     ", e.ev_desc"
 				     " FROM chain_events e"
 				     " LEFT OUTER JOIN accounts a"
 				     " ON e.account_id = a.id"
@@ -171,6 +182,7 @@ struct chain_event **account_get_chain_events(const tal_t *ctx,
 				     ", e.payment_id"
 				     ", e.ignored"
 				     ", e.stealable"
+				     ", e.ev_desc"
 				     " FROM chain_events e"
 				     " LEFT OUTER JOIN accounts a"
 				     " ON e.account_id = a.id"
@@ -205,6 +217,7 @@ static struct chain_event **find_txos_for_tx(const tal_t *ctx,
 				     ", e.payment_id"
 				     ", e.ignored"
 				     ", e.stealable"
+				     ", e.ev_desc"
 				     " FROM chain_events e"
 				     " LEFT OUTER JOIN accounts a"
 				     " ON e.account_id = a.id"
@@ -586,6 +599,31 @@ void maybe_mark_account_onchain(struct db *db, struct account *acct)
 	tal_free(ctx);
 }
 
+void add_payment_hash_desc(struct db *db,
+			   struct sha256 *payment_hash,
+			   const char *desc)
+{
+	struct db_stmt *stmt;
+
+	/* Ok, now we update the account with this blockheight */
+	stmt = db_prepare_v2(db, SQL("UPDATE channel_events SET"
+				     "  ev_desc = ?"
+				     " WHERE"
+				     " payment_id = ?"));
+	db_bind_text(stmt, 0, desc);
+	db_bind_sha256(stmt, 1, payment_hash);
+	db_exec_prepared_v2(take(stmt));
+
+	/* Ok, now we update the account with this blockheight */
+	stmt = db_prepare_v2(db, SQL("UPDATE chain_events SET"
+				     "  ev_desc = ?"
+				     " WHERE"
+				     " payment_id = ?"));
+	db_bind_text(stmt, 0, desc);
+	db_bind_sha256(stmt, 1, payment_hash);
+	db_exec_prepared_v2(take(stmt));
+}
+
 struct chain_event *find_chain_event_by_id(const tal_t *ctx,
 					   struct db *db,
 					   u64 event_db_id)
@@ -611,6 +649,7 @@ struct chain_event *find_chain_event_by_id(const tal_t *ctx,
 				     ", e.payment_id"
 				     ", e.ignored"
 				     ", e.stealable"
+				     ", e.ev_desc"
 				     " FROM chain_events e"
 				     " LEFT OUTER JOIN accounts a"
 				     " ON e.account_id = a.id"
@@ -657,6 +696,7 @@ static struct chain_event *find_chain_event(const tal_t *ctx,
 					     ", e.payment_id"
 					     ", e.ignored"
 					     ", e.stealable"
+					     ", e.ev_desc"
 					     " FROM chain_events e"
 					     " LEFT OUTER JOIN accounts a"
 					     " ON e.account_id = a.id"
@@ -685,6 +725,7 @@ static struct chain_event *find_chain_event(const tal_t *ctx,
 					     ", e.payment_id"
 					     ", e.ignored"
 					     ", e.stealable"
+					     ", e.ev_desc"
 					     " FROM chain_events e"
 					     " LEFT OUTER JOIN accounts a"
 					     " ON e.account_id = a.id"
@@ -836,6 +877,7 @@ struct channel_event **list_channel_events_timebox(const tal_t *ctx,
 				     ", e.payment_id"
 				     ", e.part_id"
 				     ", e.timestamp"
+				     ", e.ev_desc"
 				     " FROM channel_events e"
 				     " LEFT OUTER JOIN accounts a"
 				     " ON a.id = e.account_id"
@@ -882,6 +924,7 @@ struct channel_event **account_get_channel_events(const tal_t *ctx,
 				     ", e.payment_id"
 				     ", e.part_id"
 				     ", e.timestamp"
+				     ", e.ev_desc"
 				     " FROM channel_events e"
 				     " LEFT OUTER JOIN accounts a"
 				     " ON a.id = e.account_id"
@@ -1283,9 +1326,10 @@ void log_channel_event(struct db *db,
 				     ", payment_id"
 				     ", part_id"
 				     ", timestamp"
+				     ", ev_desc"
 				     ")"
 				     " VALUES"
-				     " (?, ?, ?, ?, ?, ?, ?, ?, ?);"));
+				     " (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"));
 
 	db_bind_u64(stmt, 0, acct->db_id);
 	db_bind_text(stmt, 1, e->tag);
@@ -1299,6 +1343,10 @@ void log_channel_event(struct db *db,
 		db_bind_null(stmt, 6);
 	db_bind_int(stmt, 7, e->part_id);
 	db_bind_u64(stmt, 8, e->timestamp);
+	if (e->desc)
+		db_bind_text(stmt, 9, e->desc);
+	else
+		db_bind_null(stmt, 9);
 
 	db_exec_prepared_v2(stmt);
 	e->db_id = db_last_insert_id_v2(stmt);
@@ -1330,6 +1378,7 @@ static struct chain_event **find_chain_events_bytxid(const tal_t *ctx, struct db
 				     ", e.payment_id"
 				     ", e.ignored"
 				     ", e.stealable"
+				     ", e.ev_desc"
 				     " FROM chain_events e"
 				     " LEFT OUTER JOIN accounts a"
 				     " ON a.id = e.account_id"
@@ -1820,9 +1869,10 @@ bool log_chain_event(struct db *db,
 				     ", spending_txid"
 				     ", ignored"
 				     ", stealable"
+				     ", ev_desc"
 				     ")"
 				     " VALUES "
-				     "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"));
+				     "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"));
 
 	db_bind_u64(stmt, 0, acct->db_id);
 	if (e->origin_acct)
@@ -1851,6 +1901,10 @@ bool log_chain_event(struct db *db,
 
 	db_bind_int(stmt, 13, e->ignored ? 1 : 0);
 	db_bind_int(stmt, 14, e->stealable ? 1 : 0);
+	if (e->desc)
+		db_bind_text(stmt, 15, e->desc);
+	else
+		db_bind_null(stmt, 15);
 	db_exec_prepared_v2(stmt);
 	e->db_id = db_last_insert_id_v2(stmt);
 	e->acct_db_id = acct->db_id;

plugins/bkpr/recorder.h

@@ -184,6 +184,14 @@ char *update_channel_onchain_fees(const tal_t *ctx,
  * The point of this is to allow us to prune data, eventually */
 void maybe_mark_account_onchain(struct db *db, struct account *acct);
 
+/* We fetch invoice desc data after the fact and then update it
+ * Updates both the chain_event and channel_event tables for all
+ * matching payment_hashes
+ * */
+void add_payment_hash_desc(struct db *db,
+			   struct sha256 *payment_hash,
+			   const char *desc);
+
 /* When we make external deposits from the wallet, we don't
  * count them until any output that was spent *into* them is
  * confirmed onchain.

plugins/bkpr/test/run-recorder.c

@@ -287,6 +287,10 @@ static bool channel_events_eq(struct channel_event *e1, struct channel_event *e2
 	CHECK(e1->part_id == e2->part_id);
 	CHECK(e1->timestamp == e2->timestamp);
 
+	CHECK((e1->desc != NULL) == (e2->desc != NULL));
+	if (e1->desc)
+		CHECK(streq(e1->desc, e2->desc));
+
 	return true;
 }
 
@@ -314,6 +318,9 @@ static bool chain_events_eq(struct chain_event *e1, struct chain_event *e2)
 	if (e1->payment_id)
 		CHECK(sha256_eq(e1->payment_id, e2->payment_id));
 
+	CHECK((e1->desc != NULL) == (e2->desc != NULL));
+	if (e1->desc)
+		CHECK(streq(e1->desc, e2->desc));
 
 	return true;
 }
@@ -335,6 +342,7 @@ static struct channel_event *make_channel_event(const tal_t *ctx,
 	ev->timestamp = 1919191;
 	ev->part_id = 19;
 	ev->tag = tag;
+	ev->desc = tal_fmt(ev, "description");
 	return ev;
 }
 
@@ -364,6 +372,7 @@ static struct chain_event *make_chain_event(const tal_t *ctx,
 	ev->blockheight = blockheight;
 	ev->ignored = false;
 	ev->stealable = false;
+	ev->desc = tal_fmt(ev, "hello hello");
 	memset(&ev->outpoint.txid, outpoint_char, sizeof(struct bitcoin_txid));
 	ev->outpoint.n = outnum;
 
@@ -880,9 +889,11 @@ static bool test_channel_event_crud(const tal_t *ctx, struct plugin *p)
 	ev1->currency = "btc";
 	ev1->timestamp = 11111;
 	ev1->part_id = 19;
+	ev1->desc = tal_strdup(ev1, "hello desc1");
 
 	/* Passing unknown tags in should be ok */
 	ev1->tag = "hello";
+	ev1->desc = tal_fmt(ev1, "desc");
 
 	ev2 = tal(ctx, struct channel_event);
 	ev2->payment_id = tal(ev2, struct sha256);
@@ -894,6 +905,7 @@ static bool test_channel_event_crud(const tal_t *ctx, struct plugin *p)
 	ev2->timestamp = 22222;
 	ev2->part_id = 0;
 	ev2->tag = tal_fmt(ev2, "deposit");
+	ev2->desc = NULL;
 
 	ev3 = tal(ctx, struct channel_event);
 	ev3->payment_id = tal(ev3, struct sha256);
@@ -905,6 +917,7 @@ static bool test_channel_event_crud(const tal_t *ctx, struct plugin *p)
 	ev3->timestamp = 33333;
 	ev3->part_id = 5;
 	ev3->tag = tal_fmt(ev3, "routed");
+	ev3->desc = NULL;
 
 	db_begin_transaction(db);
 	log_channel_event(db, acct, ev1);
@@ -972,6 +985,7 @@ static bool test_chain_event_crud(const tal_t *ctx, struct plugin *p)
 	ev1->spending_txid = tal(ctx, struct bitcoin_txid);
 	memset(ev1->spending_txid, 'C', sizeof(struct bitcoin_txid));
 	ev1->payment_id = NULL;
+	ev1->desc = tal_fmt(ev1, "description");
 
 	db_begin_transaction(db);
 	log_chain_event(db, acct, ev1);
@@ -992,6 +1006,7 @@ static bool test_chain_event_crud(const tal_t *ctx, struct plugin *p)
 	ev2->outpoint.n = 1;
 	ev2->spending_txid = NULL;
 	ev2->payment_id = tal(ctx, struct sha256);
+	ev2->desc = NULL;
 	memset(ev2->payment_id, 'B', sizeof(struct sha256));
 
 	/* Dummy event, logged to separate account */
@@ -1011,6 +1026,7 @@ static bool test_chain_event_crud(const tal_t *ctx, struct plugin *p)
 	ev3->spending_txid = tal(ctx, struct bitcoin_txid);
 	memset(ev3->spending_txid, 'D', sizeof(struct bitcoin_txid));
 	ev3->payment_id = NULL;
+	ev3->desc = NULL;
 
 	db_begin_transaction(db);
 	log_chain_event(db, acct, ev2);
@@ -1238,6 +1254,7 @@ static bool test_account_crud(const tal_t *ctx, struct plugin *p)
 	ev1->spending_txid = tal(ctx, struct bitcoin_txid);
 	memset(ev1->spending_txid, 'C', sizeof(struct bitcoin_txid));
 	ev1->payment_id = NULL;
+	ev1->desc = tal_fmt(ev1, "oh hello");
 
 	db_begin_transaction(db);
 	log_chain_event(db, acct, ev1);

tests/test_bookkeeper.py

@@ -376,3 +376,45 @@ def test_bookkeeping_onchaind_txs(node_factory, bitcoind):
     assert len(funds['channels']) == 0
     outs = sum([out['amount_msat'] for out in funds['outputs']])
     assert outs == only_one(wallet_bal['balances'])['balance_msat']
+
+
+def test_bookkeeping_descriptions(node_factory, bitcoind, chainparams):
+    """
+    When an 'invoice' type event comes through, we look up the description details
+    to include about the item. Particularly useful for CSV outputs etc.
+    """
+    l1, l2 = node_factory.line_graph(2, opts={'experimental-offers': None})
+
+    # Send l2 funds via the channel
+    bolt11_desc = "test bolt11 description"
+    l1.pay(l2, 11000000, label=bolt11_desc)
+    l1.daemon.wait_for_log('coin_move .* [(]invoice[)] 0msat -11000000msat')
+    l2.daemon.wait_for_log('coin_move .* [(]invoice[)] 11000000msat')
+
+    # Test paying an bolt11 invoice (rcvr)
+    l1_inc_ev = l1.rpc.bkpr_listincome()['income_events']
+    inv = only_one([ev for ev in l1_inc_ev if ev['tag'] == 'invoice'])
+    assert inv['description'] == bolt11_desc
+
+    # Test paying an bolt11 invoice (sender)
+    l2_inc_ev = l2.rpc.bkpr_listincome()['income_events']
+    inv = only_one([ev for ev in l2_inc_ev if ev['tag'] == 'invoice'])
+    assert inv['description'] == bolt11_desc
+
+    # Make an offer (l1)
+    bolt12_desc = "test bolt12 description"
+    offer = l1.rpc.call('offer', [100, bolt12_desc])
+    invoice = l2.rpc.call('fetchinvoice', {'offer': offer['bolt12']})
+    paid = l2.rpc.pay(invoice['invoice'])
+    l1.daemon.wait_for_log('coin_move .* [(]invoice[)] 100msat')
+    l2.daemon.wait_for_log('coin_move .* [(]invoice[)] 0msat -100msat')
+
+    # Test paying an offer (bolt12) (rcvr)
+    l1_inc_ev = l1.rpc.bkpr_listincome()['income_events']
+    inv = only_one([ev for ev in l1_inc_ev if 'payment_id' in ev and ev['payment_id'] == paid['payment_hash']])
+    assert inv['description'] == bolt12_desc
+
+    # Test paying an offer (bolt12) (sender)
+    l2_inc_ev = l2.rpc.bkpr_listincome()['income_events']
+    inv = only_one([ev for ev in l2_inc_ev if 'payment_id' in ev and ev['payment_id'] == paid['payment_hash'] and ev['tag'] == 'invoice'])
+    assert inv['description'] == bolt12_desc

tests/test_invoices.py

@@ -737,3 +737,8 @@ def test_invoice_deschash(node_factory, chainparams):
 
     with pytest.raises(RpcError, match=r'description already removed'):
         l2.rpc.delinvoice('label', "paid", desconly=True)
+
+    # desc-hashes lands in bookkeeper data (description)
+    wait_for(lambda: len([ev for ev in l1.rpc.bkpr_listincome()['income_events'] if ev['tag'] == 'invoice']) == 1)
+    inv = only_one([ev for ev in l1.rpc.bkpr_listincome()['income_events'] if ev['tag'] == 'invoice'])
+    assert inv['description'] == b11['description_hash']