lightningd: add ordering and pagination to listforwards.

Changelog-Added: JSON-RPC: `listforwards` new parameters `index`, `start` and `limit`.

.msggen.json

@@ -92,6 +92,10 @@
             "legacy": 0,
             "tlv": 1
         },
+        "ListforwardsIndex": {
+            "created": 0,
+            "updated": 1
+        },
         "ListforwardsStatus": {
             "failed": 3,
             "local_failed": 2,
@@ -940,7 +944,10 @@
         },
         "ListforwardsRequest": {
             "ListForwards.in_channel": 2,
+            "ListForwards.index": 4,
+            "ListForwards.limit": 6,
             "ListForwards.out_channel": 3,
+            "ListForwards.start": 5,
             "ListForwards.status": 1
         },
         "ListforwardsResponse": {
@@ -3624,10 +3631,22 @@
             "added": "pre-v0.10.1",
             "deprecated": false
         },
+        "ListForwards.index": {
+            "added": "v23.11",
+            "deprecated": false
+        },
+        "ListForwards.limit": {
+            "added": "v23.11",
+            "deprecated": false
+        },
         "ListForwards.out_channel": {
             "added": "pre-v0.10.1",
             "deprecated": false
         },
+        "ListForwards.start": {
+            "added": "v23.11",
+            "deprecated": false
+        },
         "ListForwards.status": {
             "added": "pre-v0.10.1",
             "deprecated": false

cln-grpc/proto/node.proto

@@ -1624,9 +1624,17 @@ message ListforwardsRequest {
 		LOCAL_FAILED = 2;
 		FAILED = 3;
 	}
+	// ListForwards.index
+	enum ListforwardsIndex {
+		CREATED = 0;
+		UPDATED = 1;
+	}
 	optional ListforwardsStatus status = 1;
 	optional string in_channel = 2;
 	optional string out_channel = 3;
+	optional ListforwardsIndex index = 4;
+	optional uint64 start = 5;
+	optional uint32 limit = 6;
 }
 
 message ListforwardsResponse {

cln-grpc/src/convert.rs

@@ -2315,6 +2315,9 @@ impl From<requests::ListforwardsRequest> for pb::ListforwardsRequest {
             status: c.status.map(|v| v as i32),
             in_channel: c.in_channel.map(|v| v.to_string()), // Rule #2 for type short_channel_id?
             out_channel: c.out_channel.map(|v| v.to_string()), // Rule #2 for type short_channel_id?
+            index: c.index.map(|v| v as i32),
+            start: c.start, // Rule #2 for type u64?
+            limit: c.limit, // Rule #2 for type u32?
         }
     }
 }
@@ -3023,6 +3026,9 @@ impl From<pb::ListforwardsRequest> for requests::ListforwardsRequest {
             status: c.status.map(|v| v.try_into().unwrap()),
             in_channel: c.in_channel.map(|v| cln_rpc::primitives::ShortChannelId::from_str(&v).unwrap()), // Rule #1 for type short_channel_id?
             out_channel: c.out_channel.map(|v| cln_rpc::primitives::ShortChannelId::from_str(&v).unwrap()), // Rule #1 for type short_channel_id?
+            index: c.index.map(|v| v.try_into().unwrap()),
+            start: c.start, // Rule #1 for type u64?
+            limit: c.limit, // Rule #1 for type u32?
         }
     }
 }

cln-rpc/src/model.rs

@@ -1379,6 +1379,34 @@ pub mod requests {
 	    }
 	}
 
+	#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
+	pub enum ListforwardsIndex {
+	    #[serde(rename = "created")]
+	    CREATED,
+	    #[serde(rename = "updated")]
+	    UPDATED,
+	}
+
+	impl TryFrom<i32> for ListforwardsIndex {
+	    type Error = anyhow::Error;
+	    fn try_from(c: i32) -> Result<ListforwardsIndex, anyhow::Error> {
+	        match c {
+	    0 => Ok(ListforwardsIndex::CREATED),
+	    1 => Ok(ListforwardsIndex::UPDATED),
+	            o => Err(anyhow::anyhow!("Unknown variant {} for enum ListforwardsIndex", o)),
+	        }
+	    }
+	}
+
+	impl ToString for ListforwardsIndex {
+	    fn to_string(&self) -> String {
+	        match self {
+	            ListforwardsIndex::CREATED => "CREATED",
+	            ListforwardsIndex::UPDATED => "UPDATED",
+	        }.to_string()
+	    }
+	}
+
 	#[derive(Clone, Debug, Deserialize, Serialize)]
 	pub struct ListforwardsRequest {
 	    #[serde(skip_serializing_if = "Option::is_none")]
@@ -1387,6 +1415,12 @@ pub mod requests {
 	    pub in_channel: Option<ShortChannelId>,
 	    #[serde(skip_serializing_if = "Option::is_none")]
 	    pub out_channel: Option<ShortChannelId>,
+	    #[serde(skip_serializing_if = "Option::is_none")]
+	    pub index: Option<ListforwardsIndex>,
+	    #[serde(skip_serializing_if = "Option::is_none")]
+	    pub start: Option<u64>,
+	    #[serde(skip_serializing_if = "Option::is_none")]
+	    pub limit: Option<u32>,
 	}
 
 	impl From<ListforwardsRequest> for Request {

contrib/pyln-client/pyln/client/lightning.py

@@ -1007,7 +1007,7 @@ class LightningRpc(UnixDomainSocketRpc):
         }
         return self.call("listdatastore", payload)
 
-    def listforwards(self, status=None, in_channel=None, out_channel=None):
+    def listforwards(self, status=None, in_channel=None, out_channel=None, index=None, start=None, limit=None):
         """List all forwarded payments and their information matching
         forward {status}, {in_channel} and {out_channel}.
         """
@@ -1015,6 +1015,9 @@ class LightningRpc(UnixDomainSocketRpc):
             "status": status,
             "in_channel": in_channel,
             "out_channel": out_channel,
+            "index": index,
+            "start": start,
+            "limit": limit,
         }
         return self.call("listforwards", payload)
 

doc/lightning-listforwards.7.md

@@ -4,7 +4,7 @@ lightning-listforwards -- Command showing all htlcs and their information
 SYNOPSIS
 --------
 
-**listforwards** [*status*] [*in\_channel*] [*out\_channel*]
+**listforwards** [*status*] [*in\_channel*] [*out\_channel*] [*index* [*start*] [*limit*]]
 
 DESCRIPTION
 -----------
@@ -18,6 +18,12 @@ If *status* is specified, then only the forwards with the given status are retur
 If *in\_channel* or *out\_channel* is specified, then only the matching forwards
 on the given in/out channel are returned.
 
+If neither *in\_channel* or *out\_channel* is specified,
+`index` controls ordering, by `created` (default) or `updated`.  If
+`index` is specified, `start` may be specified to start from that
+value, which is generally returned from lightning-wait(7), and `limit`
+can be used to specify the maximum number of entries to return.
+
 RETURN VALUE
 ------------
 

doc/schemas/listforwards.request.json

@@ -17,6 +17,25 @@
     },
     "out_channel": {
       "type": "short_channel_id"
+    },
+    "index": {
+      "type": "string",
+      "added": "v23.11",
+      "enum": [
+        "created",
+        "updated"
+      ],
+      "description": ""
+    },
+    "start": {
+      "type": "u64",
+      "added": "v23.11",
+      "description": ""
+    },
+    "limit": {
+      "type": "u32",
+      "added": "v23.11",
+      "description": ""
     }
   }
 }

lightningd/forwards.c

@@ -149,11 +149,14 @@ static void listforwardings_add_forwardings(struct json_stream *response,
 					    struct wallet *wallet,
 					    enum forward_status status,
 					    const struct short_channel_id *chan_in,
-					    const struct short_channel_id *chan_out)
+					    const struct short_channel_id *chan_out,
+					    const enum wait_index *listindex,
+					    u64 liststart,
+					    const u32 *listlimit)
 {
 	const struct forwarding *forwardings;
 
-	forwardings = wallet_forwarded_payments_get(wallet, tmpctx, status, chan_in, chan_out);
+	forwardings = wallet_forwarded_payments_get(wallet, tmpctx, status, chan_in, chan_out, listindex, liststart, listlimit);
 
 	json_array_start(response, "forwards");
 	for (size_t i=0; i<tal_count(forwardings); i++) {
@@ -190,17 +193,37 @@ static struct command_result *json_listforwards(struct command *cmd,
 	struct json_stream *response;
 	struct short_channel_id *chan_in, *chan_out;
 	enum forward_status *status;
+	enum wait_index *listindex;
+	u64 *liststart;
+	u32 *listlimit;
 
 	if (!param(cmd, buffer, params,
 		   p_opt_def("status", param_forward_status, &status,
 			     FORWARD_ANY),
 		   p_opt("in_channel", param_short_channel_id, &chan_in),
 		   p_opt("out_channel", param_short_channel_id, &chan_out),
+		   p_opt("index", param_index, &listindex),
+		   p_opt_def("start", param_u64, &liststart, 0),
+		   p_opt("limit", param_u32, &listlimit),
 		   NULL))
 		return command_param_failed();
 
+	if (*liststart != 0 && !listindex) {
+		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+				    "Can only specify {start} with {index}");
+	}
+	if (listlimit && !listindex) {
+		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+				    "Can only specify {limit} with {index}");
+	}
+
+	if ((chan_in || chan_out) && *liststart != 0) {
+		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+				    "Cannot use start with in_channel or out_channel");
+	}
+
 	response = json_stream_success(cmd);
-	listforwardings_add_forwardings(response, cmd->ld->wallet, *status, chan_in, chan_out);
+	listforwardings_add_forwardings(response, cmd->ld->wallet, *status, chan_in, chan_out, listindex, *liststart, listlimit);
 
 	return command_success(cmd, response);
 }

wallet/wallet.c

@@ -4832,7 +4832,10 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 						       const tal_t *ctx,
 						       enum forward_status status,
 						       const struct short_channel_id *chan_in,
-						       const struct short_channel_id *chan_out)
+						       const struct short_channel_id *chan_out,
+						       const enum wait_index *listindex,
+						       u64 liststart,
+						       const u32 *listlimit)
 {
 	struct forwarding *results = tal_arr(ctx, struct forwarding, 0);
 	size_t count = 0;
@@ -4841,57 +4844,135 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 	// placeholder for any parameter, the value doesn't matter because it's discarded by sql
 	const int any = -1;
 
-	stmt = db_prepare_v2(
-	    w->db,
-	    SQL("SELECT"
-		"  state"
-		", in_msatoshi"
-		", out_msatoshi"
-		", in_channel_scid"
-		", out_channel_scid"
-		", in_htlc_id"
-		", out_htlc_id"
-		", received_time"
-		", resolved_time"
-		", failcode "
-		", forward_style "
-		", rowid "
-		", updated_index "
-		"FROM forwards "
-		"WHERE (1 = ? OR state = ?) AND "
-		"(1 = ? OR in_channel_scid = ?) AND "
-		"(1 = ? OR out_channel_scid = ?)"));
+	/* We don't support start/limits with this */
+	if (chan_in || chan_out) {
+		stmt = db_prepare_v2(
+			w->db,
+			SQL("SELECT"
+			    "  state"
+			    ", in_msatoshi"
+			    ", out_msatoshi"
+			    ", in_channel_scid"
+			    ", out_channel_scid"
+			    ", in_htlc_id"
+			    ", out_htlc_id"
+			    ", received_time"
+			    ", resolved_time"
+			    ", failcode "
+			    ", forward_style "
+			    ", rowid "
+			    ", updated_index "
+			    "FROM forwards "
+			    "WHERE (1 = ? OR state = ?) AND "
+			    "(1 = ? OR in_channel_scid = ?) AND "
+			    "(1 = ? OR out_channel_scid = ?)"));
 
-	if (status == FORWARD_ANY) {
-		// any status
-		db_bind_int(stmt, 1);
-		db_bind_int(stmt, any);
+		if (status == FORWARD_ANY) {
+			// any status
+			db_bind_int(stmt, 1);
+			db_bind_int(stmt, any);
+		} else {
+			// specific forward status
+			db_bind_int(stmt, 0);
+			db_bind_int(stmt, status);
+		}
+
+		if (chan_in) {
+			// specific in_channel
+			db_bind_int(stmt, 0);
+			db_bind_short_channel_id(stmt, chan_in);
+		} else {
+			// any in_channel
+			db_bind_int(stmt, 1);
+			db_bind_int(stmt, any);
+		}
+
+		if (chan_out) {
+			// specific out_channel
+			db_bind_int(stmt, 0);
+			db_bind_short_channel_id(stmt, chan_out);
+		} else {
+			// any out_channel
+			db_bind_int(stmt, 1);
+			db_bind_int(stmt, any);
+		}
+	} else if (listindex && *listindex == WAIT_INDEX_UPDATED) {
+		stmt = db_prepare_v2(
+			w->db,
+			SQL("SELECT"
+			    "  state"
+			    ", in_msatoshi"
+			    ", out_msatoshi"
+			    ", in_channel_scid"
+			    ", out_channel_scid"
+			    ", in_htlc_id"
+			    ", out_htlc_id"
+			    ", received_time"
+			    ", resolved_time"
+			    ", failcode "
+			    ", forward_style "
+			    ", rowid "
+			    ", updated_index "
+			    "FROM forwards "
+			    " WHERE"
+			    "  (1 = ? OR state = ?)"
+			    " AND"
+			    "  updated_index >= ?"
+			    " ORDER BY updated_index"
+			    " LIMIT ?;"));
+		if (status == FORWARD_ANY) {
+			// any status
+			db_bind_int(stmt, 1);
+			db_bind_int(stmt, any);
+		} else {
+			// specific forward status
+			db_bind_int(stmt, 0);
+			db_bind_int(stmt, status);
+		}
+		db_bind_u64(stmt, liststart);
+		if (listlimit)
+			db_bind_int(stmt, *listlimit);
+		else
+			db_bind_int(stmt, INT_MAX);
 	} else {
-		// specific forward status
-		db_bind_int(stmt, 0);
-		db_bind_int(stmt, status);
+		stmt = db_prepare_v2(
+			w->db,
+			SQL("SELECT"
+			    "  state"
+			    ", in_msatoshi"
+			    ", out_msatoshi"
+			    ", in_channel_scid"
+			    ", out_channel_scid"
+			    ", in_htlc_id"
+			    ", out_htlc_id"
+			    ", received_time"
+			    ", resolved_time"
+			    ", failcode "
+			    ", forward_style "
+			    ", rowid "
+			    ", updated_index "
+			    "FROM forwards "
+			    " WHERE"
+			    "  (1 = ? OR state = ?)"
+			    " AND"
+			    "  rowid >= ?"
+			    " ORDER BY rowid"
+			    " LIMIT ?;"));
+		if (status == FORWARD_ANY) {
+			// any status
+			db_bind_int(stmt, 1);
+			db_bind_int(stmt, any);
+		} else {
+			// specific forward status
+			db_bind_int(stmt, 0);
+			db_bind_int(stmt, status);
+		}
+		db_bind_u64(stmt, liststart);
+		if (listlimit)
+			db_bind_int(stmt, *listlimit);
+		else
+			db_bind_int(stmt, INT_MAX);
 	}
-
-	if (chan_in) {
-		// specific in_channel
-		db_bind_int(stmt, 0);
-		db_bind_short_channel_id(stmt, chan_in);
-	} else {
-		// any in_channel
-		db_bind_int(stmt, 1);
-		db_bind_int(stmt, any);
-	}
-
-	if (chan_out) {
-		// specific out_channel
-		db_bind_int(stmt, 0);
-		db_bind_short_channel_id(stmt, chan_out);
-	} else {
-		// any out_channel
-		db_bind_int(stmt, 1);
-		db_bind_int(stmt, any);
-	}
-
 	db_query_prepared(stmt);
 
 	for (count=0; db_step(stmt); count++) {

wallet/wallet.h

@@ -1208,7 +1208,10 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 						       const tal_t *ctx,
 						       enum forward_status state,
 						       const struct short_channel_id *chan_in,
-						       const struct short_channel_id *chan_out);
+						       const struct short_channel_id *chan_out,
+						       const enum wait_index *listindex,
+						       u64 liststart,
+						       const u32 *listlimit);
 
 /**
  * Delete a particular forward entry