bkpr: add payment_id parameter to listaccountevents to filter events.

When you have *lots* of events in your bkpr database looking up a
specific event via calling bkpr-listaccountevents and using jq or
grep to filter gets very slow (and wasteful of CPU and disk resources).
This commit adds the paremeter payment_id to the call to filter for a
specific payment id via a where clause in the request to the database of bkpr.

Changelog-Added: Plugins: Add payment_id parameter to bkpr-listaccountevents to filter events.

.msggen.json

@@ -685,7 +685,8 @@
             "Bkpr-ListAccountEvents.events[].type": 2
         },
         "Bkpr-listaccounteventsRequest": {
-            "Bkpr-ListAccountEvents.account": 1
+            "Bkpr-ListAccountEvents.account": 1,
+            "Bkpr-ListAccountEvents.payment_id": 2
         },
         "Bkpr-listaccounteventsResponse": {
             "Bkpr-ListAccountEvents.events[]": 1
@@ -3872,6 +3873,10 @@
             "added": "pre-v0.10.1",
             "deprecated": null
         },
+        "Bkpr-ListAccountEvents.payment_id": {
+            "added": "v24.08",
+            "deprecated": null
+        },
         "Bkpr-ListBalances": {
             "added": "pre-v0.10.1",
             "deprecated": null

bitcoin/tx.c

@@ -690,18 +690,6 @@ fail:
 	return NULL;
 }
 
-/* <sigh>.  Bitcoind represents hashes as little-endian for RPC. */
-static void reverse_bytes(u8 *arr, size_t len)
-{
-	unsigned int i;
-
-	for (i = 0; i < len / 2; i++) {
-		unsigned char tmp = arr[i];
-		arr[i] = arr[len - 1 - i];
-		arr[len - 1 - i] = tmp;
-	}
-}
-
 bool bitcoin_txid_from_hex(const char *hexstr, size_t hexstr_len,
 			   struct bitcoin_txid *txid)
 {

bitcoin/tx.h

@@ -79,6 +79,18 @@ struct bitcoin_tx *clone_bitcoin_tx(const tal_t *ctx,
 struct bitcoin_tx *bitcoin_tx_from_hex(const tal_t *ctx, const char *hex,
 				       size_t hexlen);
 
+/* <sigh>.  Bitcoind represents hashes as little-endian for RPC. */
+static inline void reverse_bytes(u8 *arr, size_t len)
+{
+	unsigned int i;
+
+	for (i = 0; i < len / 2; i++) {
+		unsigned char tmp = arr[i];
+		arr[i] = arr[len - 1 - i];
+		arr[len - 1 - i] = tmp;
+	}
+}
+
 /* Parse hex string to get txid (reversed, a-la bitcoind). */
 bool bitcoin_txid_from_hex(const char *hexstr, size_t hexstr_len,
 			   struct bitcoin_txid *txid);

contrib/msggen/msggen/schema.json

@@ -2306,6 +2306,10 @@
         "",
         "If the optional parameter **account** is set, we only emit events for the specified account, if exists.",
         "",
+        "If the optional parameter **payment_id** is set, we only emit events which have that value as payment hash or as transaction id.",
+        "",
+        "The parameters **account** and **payment_id** are mutually exclusive.",
+        "",
         "Note that the type **onchain_fees** that are emitted are of opposite credit/debit than as they appear in **listincome**, as **listincome** shows all events from the perspective of the node, whereas **listaccountevents** just dumps the event data as we've got it. Onchain fees are updated/recorded as we get more information about input and output spends -- the total onchain fees that were recorded for a transaction for an account can be found by summing all onchain fee events and taking the difference between the **credit_msat** and **debit_msat** for these events. We do this so that successive calls to **listaccountevents** always produce the same list of events -- no previously emitted event will be subsequently updated, rather we add a new event to the list."
       ],
       "request": {
@@ -2316,6 +2320,13 @@
             "description": [
               "Receive events for the specified account."
             ]
+          },
+          "payment_id": {
+            "type": "string",
+            "added": "v24.08",
+            "description": [
+              "Receive events for the specified payment id."
+            ]
           }
         }
       },

doc/schemas/lightning-bkpr-listaccountevents.json

@@ -9,6 +9,10 @@
     "",
     "If the optional parameter **account** is set, we only emit events for the specified account, if exists.",
     "",
+    "If the optional parameter **payment_id** is set, we only emit events which have that value as payment hash or as transaction id.",
+    "",
+    "The parameters **account** and **payment_id** are mutually exclusive.",
+    "",
     "Note that the type **onchain_fees** that are emitted are of opposite credit/debit than as they appear in **listincome**, as **listincome** shows all events from the perspective of the node, whereas **listaccountevents** just dumps the event data as we've got it. Onchain fees are updated/recorded as we get more information about input and output spends -- the total onchain fees that were recorded for a transaction for an account can be found by summing all onchain fee events and taking the difference between the **credit_msat** and **debit_msat** for these events. We do this so that successive calls to **listaccountevents** always produce the same list of events -- no previously emitted event will be subsequently updated, rather we add a new event to the list."
   ],
   "request": {
@@ -19,6 +23,13 @@
         "description": [
           "Receive events for the specified account."
         ]
+      },
+      "payment_id": {
+        "type": "string",
+        "added": "v24.08",
+        "description": [
+          "Receive events for the specified payment id."
+        ]
       }
     }
   },

plugins/bkpr/bookkeeper.c

@@ -359,6 +359,8 @@ static struct command_result *json_list_account_events(struct command *cmd,
 {
 	struct json_stream *res;
 	struct account *acct;
+	struct sha256 *payment_id;
+	struct bitcoin_txid *tx_id;
 	const char *acct_name;
 	struct channel_event **channel_events;
 	struct chain_event **chain_events;
@@ -366,9 +368,16 @@ static struct command_result *json_list_account_events(struct command *cmd,
 
 	if (!param(cmd, buf, params,
 		   p_opt("account", param_string, &acct_name),
+		   p_opt("payment_id", param_sha256, &payment_id),
 		   NULL))
 		return command_param_failed();
 
+	if (acct_name && payment_id != NULL) {
+		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+				    "Can only specify one of "
+				    "{account} or {payment_id}");
+	}
+
 	if (acct_name) {
 		db_begin_transaction(db);
 		acct = find_account(cmd, db, acct_name);
@@ -386,6 +395,16 @@ static struct command_result *json_list_account_events(struct command *cmd,
 		channel_events = account_get_channel_events(cmd, db, acct);
 		chain_events = account_get_chain_events(cmd, db, acct);
 		onchain_fees = account_get_chain_fees(cmd, db, acct);
+	} else if (payment_id != NULL) {
+		channel_events = get_channel_events_by_id(cmd, db, payment_id);
+
+		tx_id = tal(cmd, struct bitcoin_txid);
+		tx_id->shad.sha = *payment_id;
+		/* Transaction ids are stored as big-endian in the database */
+		reverse_bytes(tx_id->shad.sha.u.u8, sizeof(tx_id->shad.sha.u.u8));
+
+		chain_events = find_chain_events_bytxid(cmd, db, tx_id);
+		onchain_fees = get_chain_fees_by_txid(cmd, db, tx_id);
 	} else {
 		channel_events = list_channel_events(cmd, db);
 		chain_events = list_chain_events(cmd, db);

plugins/bkpr/recorder.c

@@ -124,6 +124,26 @@ static struct channel_event *stmt2channel_event(const tal_t *ctx, struct db_stmt
 	return e;
 }
 
+static struct channel_event **find_channel_events(const tal_t *ctx,
+						  struct db_stmt *stmt TAKES)
+{
+	struct channel_event **results;
+
+	db_query_prepared(stmt);
+	if (stmt->error)
+		db_fatal(stmt->db, "find_channel_events err: %s", stmt->error);
+	results = tal_arr(ctx, struct channel_event *, 0);
+	while (db_step(stmt)) {
+		struct channel_event *e = stmt2channel_event(results, stmt);
+		tal_arr_expand(&results, e);
+	}
+
+	if (taken(stmt))
+		tal_free(stmt);
+
+	return results;
+}
+
 static struct rebalance *stmt2rebalance(const tal_t *ctx, struct db_stmt *stmt)
 {
 	struct rebalance *r = tal(ctx, struct rebalance);
@@ -956,7 +976,6 @@ struct channel_event **account_get_channel_events(const tal_t *ctx,
 						  struct account *acct)
 {
 	struct db_stmt *stmt;
-	struct channel_event **results;
 
 	stmt = db_prepare_v2(db, SQL("SELECT"
 				     "  e.id"
@@ -979,16 +998,37 @@ struct channel_event **account_get_channel_events(const tal_t *ctx,
 				     " ORDER BY e.timestamp, e.id"));
 
 	db_bind_u64(stmt, acct->db_id);
-	db_query_prepared(stmt);
+	return find_channel_events(ctx, take(stmt));
+}
 
-	results = tal_arr(ctx, struct channel_event *, 0);
-	while (db_step(stmt)) {
-		struct channel_event *e = stmt2channel_event(results, stmt);
-		tal_arr_expand(&results, e);
-	}
-	tal_free(stmt);
+struct channel_event **get_channel_events_by_id(const tal_t *ctx,
+						struct db *db,
+						struct sha256 *id)
+{
+	struct db_stmt *stmt;
 
-	return results;
+	stmt = db_prepare_v2(db, SQL("SELECT"
+				     "  e.id"
+				     ", a.name"
+				     ", e.account_id"
+				     ", e.tag"
+				     ", e.credit"
+				     ", e.debit"
+				     ", e.fees"
+				     ", e.currency"
+				     ", e.payment_id"
+				     ", e.part_id"
+				     ", e.timestamp"
+				     ", e.ev_desc"
+				     ", e.rebalance_id"
+				     " FROM channel_events e"
+				     " LEFT OUTER JOIN accounts a"
+				     " ON a.id = e.account_id"
+				     " WHERE e.payment_id = ?"
+				     " ORDER BY e.timestamp, e.id"));
+
+	db_bind_sha256(stmt, id);
+	return find_channel_events(ctx, take(stmt));
 }
 
 static struct onchain_fee *stmt2onchain_fee(const tal_t *ctx,
@@ -1008,11 +1048,30 @@ static struct onchain_fee *stmt2onchain_fee(const tal_t *ctx,
 	return of;
 }
 
+static struct onchain_fee **find_onchain_fees(const tal_t *ctx,
+					      struct db_stmt *stmt TAKES)
+{
+	struct onchain_fee **results;
+
+	db_query_prepared(stmt);
+	if (stmt->error)
+		db_fatal(stmt->db, "find_onchain_fees err: %s", stmt->error);
+	results = tal_arr(ctx, struct onchain_fee *, 0);
+	while (db_step(stmt)) {
+		struct onchain_fee *of = stmt2onchain_fee(results, stmt);
+		tal_arr_expand(&results, of);
+	}
+
+	if (taken(stmt))
+		tal_free(stmt);
+
+	return results;
+}
+
 struct onchain_fee **account_get_chain_fees(const tal_t *ctx, struct db *db,
 					    struct account *acct)
 {
 	struct db_stmt *stmt;
-	struct onchain_fee **results;
 
 	stmt = db_prepare_v2(db, SQL("SELECT"
 				     "  of.account_id"
@@ -1033,23 +1092,40 @@ struct onchain_fee **account_get_chain_fees(const tal_t *ctx, struct db *db,
 				     ", of.update_count"));
 
 	db_bind_u64(stmt, acct->db_id);
-	db_query_prepared(stmt);
+	return find_onchain_fees(ctx, take(stmt));
+}
 
-	results = tal_arr(ctx, struct onchain_fee *, 0);
-	while (db_step(stmt)) {
-		struct onchain_fee *of = stmt2onchain_fee(results, stmt);
-		tal_arr_expand(&results, of);
-	}
-	tal_free(stmt);
+struct onchain_fee **get_chain_fees_by_txid(const tal_t *ctx, struct db *db,
+					    struct bitcoin_txid *txid)
+{
+	struct db_stmt *stmt;
 
-	return results;
+	stmt = db_prepare_v2(db, SQL("SELECT"
+				     "  of.account_id"
+				     ", a.name"
+				     ", of.txid"
+				     ", of.credit"
+				     ", of.debit"
+				     ", of.currency"
+				     ", of.timestamp"
+				     ", of.update_count"
+				     " FROM onchain_fees of"
+				     " LEFT OUTER JOIN accounts a"
+				     " ON a.id = of.account_id"
+				     " WHERE of.txid = ?"
+				     " ORDER BY "
+				     "  of.timestamp"
+				     ", of.txid"
+				     ", of.update_count"));
+
+	db_bind_txid(stmt, txid);
+	return find_onchain_fees(ctx, take(stmt));
 }
 
 struct onchain_fee **list_chain_fees_timebox(const tal_t *ctx, struct db *db,
 					     u64 start_time, u64 end_time)
 {
 	struct db_stmt *stmt;
-	struct onchain_fee **results;
 
 	stmt = db_prepare_v2(db, SQL("SELECT"
 				     "  of.account_id"
@@ -1073,16 +1149,7 @@ struct onchain_fee **list_chain_fees_timebox(const tal_t *ctx, struct db *db,
 
 	db_bind_u64(stmt, start_time);
 	db_bind_u64(stmt, end_time);
-	db_query_prepared(stmt);
-
-	results = tal_arr(ctx, struct onchain_fee *, 0);
-	while (db_step(stmt)) {
-		struct onchain_fee *of = stmt2onchain_fee(results, stmt);
-		tal_arr_expand(&results, of);
-	}
-	tal_free(stmt);
-
-	return results;
+	return find_onchain_fees(ctx, take(stmt));
 }
 
 struct onchain_fee **list_chain_fees(const tal_t *ctx, struct db *db)
@@ -1167,7 +1234,6 @@ struct onchain_fee **account_onchain_fees(const tal_t *ctx,
 					  struct account *acct)
 {
 	struct db_stmt *stmt;
-	struct onchain_fee **results;
 
 	stmt = db_prepare_v2(db, SQL("SELECT"
 				     "  of.account_id"
@@ -1184,16 +1250,7 @@ struct onchain_fee **account_onchain_fees(const tal_t *ctx,
 				     " WHERE of.account_id = ?;"));
 
 	db_bind_u64(stmt, acct->db_id);
-	db_query_prepared(stmt);
-
-	results = tal_arr(ctx, struct onchain_fee *, 0);
-	while (db_step(stmt)) {
-		struct onchain_fee *of = stmt2onchain_fee(results, stmt);
-		tal_arr_expand(&results, of);
-	}
-	tal_free(stmt);
-
-	return results;
+	return find_onchain_fees(ctx, take(stmt));
 }
 
 struct account **list_accounts(const tal_t *ctx, struct db *db)
@@ -1413,8 +1470,8 @@ void log_channel_event(struct db *db,
 	tal_free(stmt);
 }
 
-static struct chain_event **find_chain_events_bytxid(const tal_t *ctx, struct db *db,
-						     struct bitcoin_txid *txid)
+struct chain_event **find_chain_events_bytxid(const tal_t *ctx, struct db *db,
+					      struct bitcoin_txid *txid)
 {
 	struct db_stmt *stmt;
 

plugins/bkpr/recorder.h

@@ -63,6 +63,11 @@ struct channel_event **account_get_channel_events(const tal_t *ctx,
 						  struct db *db,
 						  struct account *acct);
 
+/* Get all channel events for a payment id, order by timestamp */
+struct channel_event **get_channel_events_by_id(const tal_t *ctx,
+						struct db *db,
+						struct sha256 *id);
+
 /* Get all channel events, ordered by timestamp */
 struct channel_event **list_channel_events(const tal_t *ctx, struct db *db);
 
@@ -83,6 +88,10 @@ struct chain_event **account_get_chain_events(const tal_t *ctx,
 					      struct db *db,
 					      struct account *acct);
 
+/* Get all chain events for a transaction id, order by timestamp */
+struct chain_event **find_chain_events_bytxid(const tal_t *ctx, struct db *db,
+					      struct bitcoin_txid *txid);
+
 /* Get all chain events, order by timestamp.  */
 struct chain_event **list_chain_events(const tal_t *ctx, struct db *db);
 
@@ -115,6 +124,10 @@ char *account_get_balance(const tal_t *ctx,
 struct onchain_fee **account_get_chain_fees(const tal_t *ctx, struct db *db,
 					    struct account *acct);
 
+/* Get all chain fees for a transaction id, order by timestamp */
+struct onchain_fee **get_chain_fees_by_txid(const tal_t *ctx, struct db *db,
+					    struct bitcoin_txid *txid);
+
 /* Find a chain event by its database id */
 struct chain_event *find_chain_event_by_id(const tal_t *ctx,
 					   struct db *db,