mirrors/core-lightning
1
0
Fork 0
mirror of https://github.com/ElementsProject/lightning.git synced 2025-03-15 20:09:18 +01:00
Code Issues Projects Releases Wiki Activity

bkpr: track channel rebalances, display in listincome

Browse source 
Track rebalances, and report income events for them.

Previously `listincome` would report:
	- invoice event, debit, outgoing channel
	- invoice_fee event, debit, outgoing channel
	- invoice event, credit, inbound channel

Now reports:
	- rebalance_fee, debit, outgoing channel
	(same value as invoice_fee above)

Note: only applies on channel events; if a rebalance falls to chain
we'll use the older style of accounting.

Changelog-None
This commit is contained in:
niftynei 2022-07-31 16:55:58 -05:00 • committed by Rusty Russell
parent da0b651803
commit 3fcf60ab7c
14 changed files with 339 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
doc/lightning-bkpr-listaccountevents.7.md
View file

@ -45,6 +45,7 @@ If **type** is "onchain_fee":
If **type** is "channel":
- **fees_msat** (msat, optional): Amount paid in fees
- **is_rebalance** (boolean, optional): Is this payment part of a rebalance
- **payment_id** (hex, optional): lightning payment identifier. For an htlc, this will be the preimage.
- **part_id** (u32, optional): Counter for multi-part payments
@ -66,4 +67,4 @@ RESOURCES
Main web site: <https://github.com/ElementsProject/lightning>
[comment]: # ( SHA256STAMP:f8538b1d1e6cda7cd801690e5c09741c8a843b27cc922065598914516c16d2b3)
[comment]: # ( SHA256STAMP:8568188808cb649d7182ffb628950b93b18406a0498b5b6768371bc94375e258)

4
doc/schemas/bkpr-listaccountevents.schema.json
View file

@ -165,6 +165,10 @@
"type": "msat",
"description": "Amount paid in fees"
},
"is_rebalance": {
"type": "boolean",
"description": "Is this payment part of a rebalance"
},
"payment_id": {
"type": "hex",
"description": "lightning payment identifier. For an htlc, this will be the preimage."

1
plugins/bkpr/account_entry.c
View file

@ -8,6 +8,7 @@ static const char *tags[] = {
"journal_entry",
"penalty_adj",
"invoice_fee",
"rebalance_fee",
};
const char *account_entry_tag_str(enum account_entry_tag tag)

3
plugins/bkpr/account_entry.h
View file

@ -2,11 +2,12 @@
#define LIGHTNING_PLUGINS_BKPR_ACCOUNT_ENTRY_H
#include "config.h"
#define NUM_ACCOUNT_ENTRY_TAGS (INVOICEFEE + 1)
#define NUM_ACCOUNT_ENTRY_TAGS (REBALANCEFEE + 1)
enum account_entry_tag {
JOURNAL_ENTRY = 0,
PENALTY_ADJ = 1,
INVOICEFEE = 2,
REBALANCEFEE= 3,
};
/* Convert an enum into a string */

9
plugins/bkpr/bookkeeper.c
View file

@ -1645,6 +1645,7 @@ parse_and_log_channel_move(struct command *cmd,
e->timestamp = timestamp;
e->tag = mvt_tag_str(tags[0]);
e->desc = tal_steal(e, desc);
e->rebalance_id = NULL;
/* Go find the account for this event */
db_begin_transaction(db);
@ -1656,7 +1657,6 @@ parse_and_log_channel_move(struct command *cmd,
acct_name);
log_channel_event(db, acct, e);
db_commit_transaction(db);
/* Check for invoice desc data, necessary */
if (e->payment_id) {
@ -1664,6 +1664,12 @@ parse_and_log_channel_move(struct command *cmd,
if (tags[i] != INVOICE)
continue;
/* We only do rebalance checks for debits,
* the credit event always arrives first */
if (!amount_msat_zero(e->debit))
maybe_record_rebalance(db, e);
db_commit_transaction(db);
/* Keep memleak happy */
tal_steal(tmpctx, e);
return lookup_invoice_desc(cmd, e->credit,
@ -1671,6 +1677,7 @@ parse_and_log_channel_move(struct command *cmd,
}
}
db_commit_transaction(db);
return notification_handled(cmd);
}

3
plugins/bkpr/chain_event.h
View file

@ -34,6 +34,9 @@ struct chain_event {
* we'll need to watch it for longer */
bool stealable;
/* Is this a rebalance event? */
bool rebalance;
/* Amount we received in this event */
struct amount_msat credit;

2
plugins/bkpr/channel_event.c
View file

@ -27,6 +27,7 @@ struct channel_event *new_channel_event(const tal_t *ctx,
ev->part_id = part_id;
ev->timestamp = timestamp;
ev->desc = NULL;
ev->rebalance_id = NULL;
return ev;
}
@ -50,5 +51,6 @@ void json_add_channel_event(struct json_stream *out,
json_add_u64(out, "timestamp", ev->timestamp);
if (ev->desc)
json_add_string(out, "description", ev->desc);
json_add_bool(out, "is_rebalance", ev->rebalance_id != NULL);
json_object_end(out);
}

3
plugins/bkpr/channel_event.h
View file

@ -46,6 +46,9 @@ struct channel_event {
/* Description, usually from invoice */
const char *desc;
/* ID of paired event, iff is a rebalance */
u64 *rebalance_id;
};
struct channel_event *new_channel_event(const tal_t *ctx,

1
plugins/bkpr/db.c
View file

@ -98,6 +98,7 @@ static struct migration db_migrations[] = {
{SQL("ALTER TABLE chain_events ADD stealable INTEGER;"), NULL},
{SQL("ALTER TABLE chain_events ADD ev_desc TEXT DEFAULT NULL;"), NULL},
{SQL("ALTER TABLE channel_events ADD ev_desc TEXT DEFAULT NULL;"), NULL},
{SQL("ALTER TABLE channel_events ADD rebalance_id BIGINT DEFAULT NULL;"), NULL},
};
static bool db_migrate(struct plugin *p, struct db *db, bool *created)

21
plugins/bkpr/incomestmt.c
View file

@ -222,6 +222,16 @@ static struct income_event *paid_invoice_fee(const tal_t *ctx,
return iev;
}
static struct income_event *rebalance_fee(const tal_t *ctx,
struct channel_event *ev)
{
struct income_event *iev;
iev = channel_to_income(ctx, ev, AMOUNT_MSAT(0), ev->fees);
iev->tag = tal_free(ev->tag);
iev->tag = (char *)account_entry_tag_str(REBALANCEFEE);
return iev;
}
static struct income_event *maybe_channel_income(const tal_t *ctx,
struct channel_event *ev)
{
@ -235,6 +245,10 @@ static struct income_event *maybe_channel_income(const tal_t *ctx,
}
if (streq(ev->tag, "invoice")) {
/* Skip events for rebalances */
if (ev->rebalance_id)
return NULL;
/* If it's a payment, we note fees separately */
if (!amount_msat_zero(ev->debit)) {
struct amount_msat paid;
@ -383,11 +397,14 @@ struct income_event **list_income_events(const tal_t *ctx,
if (ev)
tal_arr_expand(&evs, ev);
/* Breakout fees on sent payments, if present */
/* Report fees on payments, if present */
if (streq(chan->tag, "invoice")
&& !amount_msat_zero(chan->debit)
&& !amount_msat_zero(chan->fees)) {
ev = paid_invoice_fee(evs, chan);
if (!chan->rebalance_id)
ev = paid_invoice_fee(evs, chan);
else
ev = rebalance_fee(evs, chan);
tal_arr_expand(&evs, ev);
}

117
plugins/bkpr/recorder.c
View file

@ -112,9 +112,29 @@ static struct channel_event *stmt2channel_event(const tal_t *ctx, struct db_stmt
else
e->desc = NULL;
if (!db_col_is_null(stmt, "e.rebalance_id")) {
e->rebalance_id = tal(e, u64);
*e->rebalance_id = db_col_u64(stmt, "e.rebalance_id");
} else
e->rebalance_id = NULL;
return e;
}
static struct rebalance *stmt2rebalance(const tal_t *ctx, struct db_stmt *stmt)
{
struct rebalance *r = tal(ctx, struct rebalance);
r->in_ev_id = db_col_u64(stmt, "in_e.id");
r->out_ev_id = db_col_u64(stmt, "out_e.id");
r->in_acct_name = db_col_strdup(r, stmt, "in_acct.name");
r->out_acct_name = db_col_strdup(r, stmt, "out_acct.name");
db_col_amount_msat(stmt, "in_e.credit", &r->rebal_msat);
db_col_amount_msat(stmt, "out_e.fees", &r->fee_msat);
return r;
}
struct chain_event **list_chain_events_timebox(const tal_t *ctx,
struct db *db,
u64 start_time,
@ -889,6 +909,7 @@ struct channel_event **list_channel_events_timebox(const tal_t *ctx,
", e.part_id"
", e.timestamp"
", e.ev_desc"
", e.rebalance_id"
" FROM channel_events e"
" LEFT OUTER JOIN accounts a"
" ON a.id = e.account_id"
@ -936,6 +957,7 @@ struct channel_event **account_get_channel_events(const tal_t *ctx,
", e.part_id"
", e.timestamp"
", e.ev_desc"
", e.rebalance_id"
" FROM channel_events e"
" LEFT OUTER JOIN accounts a"
" ON a.id = e.account_id"
@ -1339,9 +1361,10 @@ void log_channel_event(struct db *db,
", part_id"
", timestamp"
", ev_desc"
", rebalance_id"
")"
" VALUES"
" (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"));
" (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"));
db_bind_u64(stmt, 0, acct->db_id);
db_bind_text(stmt, 1, e->tag);
@ -1360,6 +1383,11 @@ void log_channel_event(struct db *db,
else
db_bind_null(stmt, 9);
if (e->rebalance_id)
db_bind_u64(stmt, 10, *e->rebalance_id);
else
db_bind_null(stmt, 10);
db_exec_prepared_v2(stmt);
e->db_id = db_last_insert_id_v2(stmt);
e->acct_db_id = acct->db_id;
@ -1644,6 +1672,93 @@ static char *is_closed_channel_txid(const tal_t *ctx, struct db *db,
return NULL;
}
void maybe_record_rebalance(struct db *db,
struct channel_event *out)
{
/* If there's a matching credit event, this is
* a rebalance. Mark everything with the payment_id
* and amt as such. If you repeat a payment_id
* with the same amt, they'll be marked as rebalances
* also */
struct db_stmt *stmt;
struct amount_msat credit;
bool ok;
/* The amount of we were credited is debit - fees */
ok = amount_msat_sub(&credit, out->debit, out->fees);
assert(ok);
stmt = db_prepare_v2(db, SQL("SELECT "
" e.id"
" FROM channel_events e"
" WHERE e.payment_id = ?"
" AND e.credit = ?"
" AND e.rebalance_id IS NULL"));
db_bind_sha256(stmt, 0, out->payment_id);
db_bind_amount_msat(stmt, 1, &credit);
db_query_prepared(stmt);
if (!db_step(stmt)) {
/* No matching invoice found */
tal_free(stmt);
return;
}
/* We just take the first one */
out->rebalance_id = tal(out, u64);
*out->rebalance_id = db_col_u64(stmt, "e.id");
tal_free(stmt);
/* Set rebalance flag on both records */
stmt = db_prepare_v2(db, SQL("UPDATE channel_events SET"
" rebalance_id = ?"
" WHERE"
" id = ?"));
db_bind_u64(stmt, 0, *out->rebalance_id);
db_bind_u64(stmt, 1, out->db_id);
db_exec_prepared_v2(take(stmt));
stmt = db_prepare_v2(db, SQL("UPDATE channel_events SET"
" rebalance_id = ?"
" WHERE"
" id = ?"));
db_bind_u64(stmt, 0, out->db_id);
db_bind_u64(stmt, 1, *out->rebalance_id);
db_exec_prepared_v2(take(stmt));
}
struct rebalance **list_rebalances(const tal_t *ctx, struct db *db)
{
struct rebalance **result;
struct db_stmt *stmt;
stmt = db_prepare_v2(db, SQL("SELECT "
" in_e.id"
", out_e.id"
", in_acct.name"
", out_acct.name"
", in_e.credit"
", out_e.fees"
" FROM channel_events in_e"
" LEFT OUTER JOIN channel_events out_e"
" ON in_e.rebalance_id = out_e.id"
" LEFT OUTER JOIN accounts out_acct"
" ON out_acct.id = out_e.account_id"
" LEFT OUTER JOIN accounts in_acct"
" ON in_acct.id = in_e.account_id"
" WHERE in_e.rebalance_id IS NOT NULL"
" AND in_e.credit > 0"));
db_query_prepared(stmt);
result = tal_arr(ctx, struct rebalance *, 0);
while (db_step(stmt)) {
struct rebalance *r = stmt2rebalance(result, stmt);
tal_arr_expand(&result, r);
}
tal_free(stmt);
return result;
}
char *maybe_update_onchain_fees(const tal_t *ctx, struct db *db,
struct bitcoin_txid *txid)
{

17
plugins/bkpr/recorder.h
View file

@ -41,6 +41,15 @@ struct txo_set {
struct txo_pair **pairs;
};
struct rebalance {
u64 in_ev_id;
u64 out_ev_id;
char *in_acct_name;
char *out_acct_name;
struct amount_msat rebal_msat;
struct amount_msat fee_msat;
};
/* Get all accounts */
struct account **list_accounts(const tal_t *ctx, struct db *db);
@ -200,6 +209,14 @@ void add_payment_hash_desc(struct db *db,
* height an input was spent into */
void maybe_closeout_external_deposits(struct db *db, struct chain_event *ev);
/* Keep track of rebalancing payments (payments paid to/from ourselves.
* Returns true if was rebalance */
void maybe_record_rebalance(struct db *db,
struct channel_event *out);
/* List all rebalances */
struct rebalance **list_rebalances(const tal_t *ctx, struct db *db);
/* Log a channel event */
void log_channel_event(struct db *db,
const struct account *acct,

97
plugins/bkpr/test/run-recorder.c
View file

@ -283,6 +283,9 @@ static bool channel_events_eq(struct channel_event *e1, struct channel_event *e2
CHECK(amount_msat_eq(e1->credit, e2->credit));
CHECK(amount_msat_eq(e1->debit, e2->debit));
CHECK(amount_msat_eq(e1->fees, e2->fees));
CHECK((e1->rebalance_id != NULL) == (e2->rebalance_id != NULL));
if (e1->rebalance_id)
CHECK(*e1->rebalance_id == *e2->rebalance_id);
CHECK(streq(e1->currency, e2->currency));
CHECK((e1->payment_id != NULL) == (e2->payment_id != NULL));
if (e1->payment_id)
@ -311,6 +314,8 @@ static bool chain_events_eq(struct chain_event *e1, struct chain_event *e2)
CHECK(streq(e1->currency, e2->currency));
CHECK(e1->timestamp == e2->timestamp);
CHECK(e1->blockheight == e2->blockheight);
CHECK(e1->stealable == e2->stealable);
CHECK(e1->ignored == e2->ignored);
CHECK(bitcoin_outpoint_eq(&e1->outpoint, &e2->outpoint));
CHECK((e1->spending_txid != NULL) == (e2->spending_txid != NULL));
@ -346,6 +351,7 @@ static struct channel_event *make_channel_event(const tal_t *ctx,
ev->part_id = 19;
ev->tag = tag;
ev->desc = tal_fmt(ev, "description");
ev->rebalance_id = NULL;
return ev;
}
@ -869,6 +875,92 @@ static bool test_onchain_fee_chan_open(const tal_t *ctx, struct plugin *p)
return true;
}
static bool test_channel_rebalances(const tal_t *ctx, struct plugin *p)
{
bool created;
struct db *db = db_setup(ctx, p, tmp_dsn(ctx), &created);
struct channel_event *ev1, *ev2, *ev3, **chan_evs;
struct rebalance **rebals;
struct account *acct1, *acct2, *acct3;
struct node_id peer_id;
memset(&peer_id, 3, sizeof(struct node_id));
acct1 = new_account(ctx, tal_fmt(ctx, "one"), &peer_id);
acct2 = new_account(ctx, tal_fmt(ctx, "two"), &peer_id);
acct3 = new_account(ctx, tal_fmt(ctx, "three"), &peer_id);
db_begin_transaction(db);
account_add(db, acct1);
account_add(db, acct2);
account_add(db, acct3);
/* Simulate a rebalance of 100msats, w/ a 12msat fee */
ev1 = make_channel_event(ctx, "invoice",
AMOUNT_MSAT(100),
AMOUNT_MSAT(0),
'A');
ev1->fees = AMOUNT_MSAT(0);
log_channel_event(db, acct1, ev1);
ev2 = make_channel_event(ctx, "invoice",
AMOUNT_MSAT(0),
AMOUNT_MSAT(112),
'A');
ev2->fees = AMOUNT_MSAT(12);
log_channel_event(db, acct2, ev2);
/* Third event w/ same preimage but diff amounts */
ev3 = make_channel_event(ctx, "invoice",
AMOUNT_MSAT(105),
AMOUNT_MSAT(0),
'A');
log_channel_event(db, acct3, ev3);
db_commit_transaction(db);
CHECK_MSG(!db_err, db_err);
db_begin_transaction(db);
chan_evs = account_get_channel_events(ctx, db, acct1);
CHECK(tal_count(chan_evs) == 1 && !chan_evs[0]->rebalance_id);
chan_evs = account_get_channel_events(ctx, db, acct2);
CHECK(tal_count(chan_evs) == 1 && !chan_evs[0]->rebalance_id);
chan_evs = account_get_channel_events(ctx, db, acct3);
CHECK(tal_count(chan_evs) == 1 && !chan_evs[0]->rebalance_id);
maybe_record_rebalance(db, ev2);
CHECK(ev2->rebalance_id != NULL);
/* Both events should be marked as rebalance */
chan_evs = account_get_channel_events(ctx, db, acct1);
CHECK(tal_count(chan_evs) == 1 && chan_evs[0]->rebalance_id);
chan_evs = account_get_channel_events(ctx, db, acct2);
CHECK(tal_count(chan_evs) == 1 && chan_evs[0]->rebalance_id);
/* Third event is not a rebalance though */
chan_evs = account_get_channel_events(ctx, db, acct3);
CHECK(tal_count(chan_evs) == 1 && !chan_evs[0]->rebalance_id);
/* Did we get an accurate rebalances entry? */
rebals = list_rebalances(ctx, db);
CHECK(tal_count(rebals) == 1);
CHECK(rebals[0]->in_ev_id == ev1->db_id);
CHECK(rebals[0]->out_ev_id == ev2->db_id);
CHECK(streq(rebals[0]->in_acct_name, "one"));
CHECK(streq(rebals[0]->out_acct_name, "two"));
CHECK(amount_msat_eq(rebals[0]->rebal_msat, AMOUNT_MSAT(100)));
CHECK(amount_msat_eq(rebals[0]->fee_msat, AMOUNT_MSAT(12)));
db_commit_transaction(db);
CHECK_MSG(!db_err, db_err);
return true;
}
static bool test_channel_event_crud(const tal_t *ctx, struct plugin *p)
{
bool created;
@ -897,6 +989,7 @@ static bool test_channel_event_crud(const tal_t *ctx, struct plugin *p)
ev1->timestamp = 11111;
ev1->part_id = 19;
ev1->desc = tal_strdup(ev1, "hello desc1");
ev1->rebalance_id = NULL;
/* Passing unknown tags in should be ok */
ev1->tag = "hello";
@ -913,6 +1006,8 @@ static bool test_channel_event_crud(const tal_t *ctx, struct plugin *p)
ev2->part_id = 0;
ev2->tag = tal_fmt(ev2, "deposit");
ev2->desc = NULL;
ev2->rebalance_id = tal(ev2, u64);
*ev2->rebalance_id = 1;
ev3 = tal(ctx, struct channel_event);
ev3->payment_id = tal(ev3, struct sha256);
@ -925,6 +1020,7 @@ static bool test_channel_event_crud(const tal_t *ctx, struct plugin *p)
ev3->part_id = 5;
ev3->tag = tal_fmt(ev3, "routed");
ev3->desc = NULL;
ev3->rebalance_id = NULL;
db_begin_transaction(db);
log_channel_event(db, acct, ev1);
@ -1322,6 +1418,7 @@ int main(int argc, char *argv[])
ok &= test_account_balances(tmpctx, plugin);
ok &= test_onchain_fee_chan_close(tmpctx, plugin);
ok &= test_onchain_fee_chan_open(tmpctx, plugin);
ok &= test_channel_rebalances(tmpctx, plugin);
ok &= test_onchain_fee_wallet_spend(tmpctx, plugin);
}

65
tests/test_bookkeeper.py
View file

@ -3,7 +3,7 @@ from decimal import Decimal
from pyln.client import Millisatoshi
from fixtures import TEST_NETWORK
from utils import (
sync_blockheight, wait_for, only_one, first_channel_id
sync_blockheight, wait_for, only_one, first_channel_id, TIMEOUT
)
from pathlib import Path
@ -571,3 +571,66 @@ def test_bookkeeping_descriptions(node_factory, bitcoind, chainparams):
l2_koinly_csv = open(koinly_path, 'rb').read()
assert l2_koinly_csv.find(bolt11_exp) >= 0
assert l2_koinly_csv.find(bolt12_exp) >= 0
def test_rebalance_tracking(node_factory, bitcoind):
"""
We identify rebalances (invoices paid and received by our node),
this allows us to filter them out of "incomes" (self-transfers are not income/exp)
and instead only display the cost incurred to move the payment (correctly
marked as a rebalance)
1 -> 2 -> 3 -> 1
"""
rebal_amt = 3210
l1, l2, l3 = node_factory.get_nodes(3)
l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
l3.rpc.connect(l1.info['id'], 'localhost', l1.port)
c12, _ = l1.fundchannel(l2, 10**7, wait_for_active=True)
c23, _ = l2.fundchannel(l3, 10**7, wait_for_active=True)
c31, _ = l3.fundchannel(l1, 10**7, wait_for_active=True)
# Build a rebalance payment
invoice = l1.rpc.invoice(rebal_amt, 'to_self', 'to_self')
pay_hash = invoice['payment_hash']
pay_sec = invoice['payment_secret']
route = [{
'id': l2.info['id'],
'channel': c12,
'direction': int(not l1.info['id'] < l2.info['id']),
'amount_msat': rebal_amt + 1001,
'style': 'tlv',
'delay': 24,
}, {
'id': l3.info['id'],
'channel': c23,
'direction': int(not l2.info['id'] < l3.info['id']),
'amount_msat': rebal_amt + 500,
'style': 'tlv',
'delay': 16,
}, {
'id': l1.info['id'],
'channel': c31,
'direction': int(not l3.info['id'] < l1.info['id']),
'amount_msat': rebal_amt,
'style': 'tlv',
'delay': 8,
}]
l1.rpc.sendpay(route, pay_hash, payment_secret=pay_sec)
result = l1.rpc.waitsendpay(pay_hash, TIMEOUT)
assert result['status'] == 'complete'
wait_for(lambda: 'invoice' not in [ev['tag'] for ev in l1.rpc.bkpr_listincome()['income_events']])
inc_evs = l1.rpc.bkpr_listincome()['income_events']
outbound_chan_id = only_one(only_one(l1.rpc.listpeers(l2.info['id'])['peers'])['channels'])['channel_id']
outbound_ev = only_one([ev for ev in inc_evs if ev['tag'] == 'rebalance_fee'])
assert outbound_ev['account'] == outbound_chan_id
assert outbound_ev['debit_msat'] == Millisatoshi(1001)
assert outbound_ev['credit_msat'] == Millisatoshi(0)
assert outbound_ev['payment_id'] == pay_hash