diff --git a/channeldb/payments.go b/channeldb/payments.go index 8c2a9507a..e1febf70d 100644 --- a/channeldb/payments.go +++ b/channeldb/payments.go @@ -532,6 +532,10 @@ type PaymentsQuery struct { // fully completed. This means that pending payments, as well as failed // payments will show up if this field is set to true. IncludeIncomplete bool + + // CountTotal indicates that all payments currently present in the + // payment index (complete and incomplete) should be counted. + CountTotal bool } // PaymentsResponse contains the result of a query to the payments database. @@ -555,6 +559,11 @@ type PaymentsResponse struct { // in the event that the slice has too many events to fit into a single // response. The offset can be used to continue forward pagination. LastIndexOffset uint64 + + // TotalCount represents the total number of payments that are currently + // stored in the payment database. This will only be set if the + // CountTotal field in the query was set to true. + TotalCount uint64 } // QueryPayments is a query to the payments database which is restricted @@ -624,6 +633,35 @@ func (d *DB) QueryPayments(query PaymentsQuery) (PaymentsResponse, error) { return err } + // Counting the total number of payments is expensive, since we + // literally have to traverse the cursor linearly, which can + // take quite a while. So it's an optional query parameter. + if query.CountTotal { + var ( + totalPayments uint64 + err error + ) + countFn := func(_, _ []byte) error { + totalPayments++ + + return nil + } + + // In non-boltdb database backends, there's a faster + // ForAll query that allows for batch fetching items. + if fastBucket, ok := indexes.(kvdb.ExtendedRBucket); ok { + err = fastBucket.ForAll(countFn) + } else { + err = indexes.ForEach(countFn) + } + if err != nil { + return fmt.Errorf("error counting payments: %v", + err) + } + + resp.TotalCount = totalPayments + } + return nil }, func() { resp = PaymentsResponse{} diff --git a/cmd/lncli/cmd_payments.go b/cmd/lncli/cmd_payments.go index 532098b6c..350905ca3 100644 --- a/cmd/lncli/cmd_payments.go +++ b/cmd/lncli/cmd_payments.go @@ -1130,13 +1130,22 @@ var listPaymentsCommand = cli.Command{ Name: "listpayments", Category: "Payments", Usage: "List all outgoing payments.", - Description: "This command enables the retrieval of payments stored " + - "in the database. Pagination is supported by the usage of " + - "index_offset in combination with the paginate_forwards flag. " + - "Reversed pagination is enabled by default to receive " + - "current payments first. Pagination can be resumed by using " + - "the returned last_index_offset (for forwards order), or " + - "first_index_offset (for reversed order) as the offset_index. ", + Description: ` + This command enables the retrieval of payments stored + in the database. + + Pagination is supported by the usage of index_offset in combination with + the paginate_forwards flag. + Reversed pagination is enabled by default to receive current payments + first. Pagination can be resumed by using the returned last_index_offset + (for forwards order), or first_index_offset (for reversed order) as the + offset_index. + + Because counting all payments in the payment database can take a long + time on systems with many payments, the count is not returned by + default. That feature can be turned on with the --count_total_payments + flag. + `, Flags: []cli.Flag{ cli.BoolFlag{ Name: "include_incomplete", @@ -1167,6 +1176,13 @@ var listPaymentsCommand = cli.Command{ "index_offset will be returned, allowing " + "forwards pagination", }, + cli.BoolFlag{ + Name: "count_total_payments", + Usage: "if set, all payments (complete or incomplete, " + + "independent of max_payments parameter) will " + + "be counted; can take a long time on systems " + + "with many payments", + }, }, Action: actionDecorator(listPayments), } @@ -1177,10 +1193,11 @@ func listPayments(ctx *cli.Context) error { defer cleanUp() req := &lnrpc.ListPaymentsRequest{ - IncludeIncomplete: ctx.Bool("include_incomplete"), - IndexOffset: uint64(ctx.Uint("index_offset")), - MaxPayments: uint64(ctx.Uint("max_payments")), - Reversed: !ctx.Bool("paginate_forwards"), + IncludeIncomplete: ctx.Bool("include_incomplete"), + IndexOffset: uint64(ctx.Uint("index_offset")), + MaxPayments: uint64(ctx.Uint("max_payments")), + Reversed: !ctx.Bool("paginate_forwards"), + CountTotalPayments: ctx.Bool("count_total_payments"), } payments, err := client.ListPayments(ctxc, req) diff --git a/docs/release-notes/release-notes-0.15.0.md b/docs/release-notes/release-notes-0.15.0.md index 898582c0f..f31db396c 100644 --- a/docs/release-notes/release-notes-0.15.0.md +++ b/docs/release-notes/release-notes-0.15.0.md @@ -245,6 +245,11 @@ from occurring that would result in an erroneous force close.](https://github.co * [Expose](https://github.com/lightningnetwork/lnd/pull/6454) always on mode of the HTLC interceptor API through GetInfo. +* [The `lnrpc.ListPayments` RPC now has an optional `count_total_payments` + parameter that will cause the `total_num_payments` response field to be set + to the total number of payments (complete and incomplete) that are currently + in the payment database](https://github.com/lightningnetwork/lnd/pull/6463). + ## Database * [Add ForAll implementation for etcd to speed up diff --git a/lnrpc/lightning.pb.go b/lnrpc/lightning.pb.go index 3506b9645..fe5d9658e 100644 --- a/lnrpc/lightning.pb.go +++ b/lnrpc/lightning.pb.go @@ -12481,6 +12481,12 @@ type ListPaymentsRequest struct { //specified index offset. This can be used to paginate backwards. The order //of the returned payments is always oldest first (ascending index order). Reversed bool `protobuf:"varint,4,opt,name=reversed,proto3" json:"reversed,omitempty"` + // + //If set, all payments (complete and incomplete, independent of the + //max_payments parameter) will be counted. Note that setting this to true will + //increase the run time of the call significantly on systems that have a lot + //of payments, as all of them have to be iterated through to be counted. + CountTotalPayments bool `protobuf:"varint,5,opt,name=count_total_payments,json=countTotalPayments,proto3" json:"count_total_payments,omitempty"` } func (x *ListPaymentsRequest) Reset() { @@ -12543,6 +12549,13 @@ func (x *ListPaymentsRequest) GetReversed() bool { return false } +func (x *ListPaymentsRequest) GetCountTotalPayments() bool { + if x != nil { + return x.CountTotalPayments + } + return false +} + type ListPaymentsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -12558,6 +12571,12 @@ type ListPaymentsResponse struct { //The index of the last item in the set of returned payments. This can be used //as the index_offset to continue seeking forwards in the next request. LastIndexOffset uint64 `protobuf:"varint,3,opt,name=last_index_offset,json=lastIndexOffset,proto3" json:"last_index_offset,omitempty"` + // + //Will only be set if count_total_payments in the request was set. Represents + //the total number of payments (complete and incomplete, independent of the + //number of payments requested in the query) currently present in the payments + //database. + TotalNumPayments uint64 `protobuf:"varint,4,opt,name=total_num_payments,json=totalNumPayments,proto3" json:"total_num_payments,omitempty"` } func (x *ListPaymentsResponse) Reset() { @@ -12613,6 +12632,13 @@ func (x *ListPaymentsResponse) GetLastIndexOffset() uint64 { return 0 } +func (x *ListPaymentsResponse) GetTotalNumPayments() uint64 { + if x != nil { + return x.TotalNumPayments + } + return 0 +} + type DeletePaymentRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -18506,7 +18532,7 @@ var file_lightning_proto_rawDesc = []byte{ 0x54, 0x4c, 0x43, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x4e, 0x5f, 0x46, 0x4c, 0x49, 0x47, 0x48, 0x54, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x55, 0x43, 0x43, 0x45, 0x45, 0x44, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, - 0x44, 0x10, 0x02, 0x22, 0xa6, 0x01, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6d, + 0x44, 0x10, 0x02, 0x22, 0xd8, 0x01, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, @@ -18516,17 +18542,23 @@ var file_lightning_proto_rawDesc = []byte{ 0x0c, 0x6d, 0x61, 0x78, 0x5f, 0x70, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x6d, 0x61, 0x78, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x76, 0x65, 0x72, 0x73, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x76, 0x65, 0x72, 0x73, 0x65, 0x64, 0x22, 0x9c, 0x01, 0x0a, - 0x14, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x08, 0x70, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x6c, 0x6e, 0x72, 0x70, 0x63, 0x2e, - 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x08, 0x70, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, - 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, - 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x66, - 0x69, 0x72, 0x73, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, - 0x2a, 0x0a, 0x11, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x6f, 0x66, - 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x6c, 0x61, 0x73, 0x74, - 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x65, 0x0a, 0x14, 0x44, + 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x76, 0x65, 0x72, 0x73, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x14, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x70, 0x61, 0x79, 0x6d, + 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xca, + 0x01, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x08, 0x70, 0x61, 0x79, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x6c, 0x6e, 0x72, 0x70, + 0x63, 0x2e, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x08, 0x70, 0x61, 0x79, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x10, 0x66, 0x69, 0x72, 0x73, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4f, 0x66, 0x66, 0x73, 0x65, + 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, + 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x6c, 0x61, + 0x73, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x2c, 0x0a, + 0x12, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x70, 0x61, 0x79, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x74, 0x6f, 0x74, 0x61, 0x6c, + 0x4e, 0x75, 0x6d, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x65, 0x0a, 0x14, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x61, 0x79, 0x6d, 0x65, diff --git a/lnrpc/lightning.proto b/lnrpc/lightning.proto index 87354ef10..7bf4703c6 100644 --- a/lnrpc/lightning.proto +++ b/lnrpc/lightning.proto @@ -3633,6 +3633,14 @@ message ListPaymentsRequest { of the returned payments is always oldest first (ascending index order). */ bool reversed = 4; + + /* + If set, all payments (complete and incomplete, independent of the + max_payments parameter) will be counted. Note that setting this to true will + increase the run time of the call significantly on systems that have a lot + of payments, as all of them have to be iterated through to be counted. + */ + bool count_total_payments = 5; } message ListPaymentsResponse { @@ -3650,6 +3658,14 @@ message ListPaymentsResponse { as the index_offset to continue seeking forwards in the next request. */ uint64 last_index_offset = 3; + + /* + Will only be set if count_total_payments in the request was set. Represents + the total number of payments (complete and incomplete, independent of the + number of payments requested in the query) currently present in the payments + database. + */ + uint64 total_num_payments = 4; } message DeletePaymentRequest { diff --git a/lnrpc/lightning.swagger.json b/lnrpc/lightning.swagger.json index 724e829eb..c825154e0 100644 --- a/lnrpc/lightning.swagger.json +++ b/lnrpc/lightning.swagger.json @@ -1916,6 +1916,13 @@ "in": "query", "required": false, "type": "boolean" + }, + { + "name": "count_total_payments", + "description": "If set, all payments (complete and incomplete, independent of the\nmax_payments parameter) will be counted. Note that setting this to true will\nincrease the run time of the call significantly on systems that have a lot\nof payments, as all of them have to be iterated through to be counted.", + "in": "query", + "required": false, + "type": "boolean" } ], "tags": [ @@ -5038,6 +5045,11 @@ "type": "string", "format": "uint64", "description": "The index of the last item in the set of returned payments. This can be used\nas the index_offset to continue seeking forwards in the next request." + }, + "total_num_payments": { + "type": "string", + "format": "uint64", + "description": "Will only be set if count_total_payments in the request was set. Represents\nthe total number of payments (complete and incomplete, independent of the\nnumber of payments requested in the query) currently present in the payments\ndatabase." } } }, diff --git a/rpcserver.go b/rpcserver.go index d034d21fa..0d5202eb3 100644 --- a/rpcserver.go +++ b/rpcserver.go @@ -6173,6 +6173,7 @@ func (r *rpcServer) ListPayments(ctx context.Context, MaxPayments: req.MaxPayments, Reversed: req.Reversed, IncludeIncomplete: req.IncludeIncomplete, + CountTotal: req.CountTotalPayments, } // If the maximum number of payments wasn't specified, then we'll @@ -6189,6 +6190,7 @@ func (r *rpcServer) ListPayments(ctx context.Context, paymentsResp := &lnrpc.ListPaymentsResponse{ LastIndexOffset: paymentsQuerySlice.LastIndexOffset, FirstIndexOffset: paymentsQuerySlice.FirstIndexOffset, + TotalNumPayments: paymentsQuerySlice.TotalCount, } for _, payment := range paymentsQuerySlice.Payments {