detect links between channel close and open txs

This commit is contained in:
Mononaut 2022-11-10 12:21:19 -06:00 committed by softsimon
parent 50993d3b95
commit cf89ded14d
No known key found for this signature in database
GPG Key ID: 488D7DCFB5A430D7
4 changed files with 402 additions and 3 deletions

View File

@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 47;
private static currentVersion = 48;
private queryTimeout = 900_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -379,7 +379,12 @@ class DatabaseMigration {
await this.$executeQuery(this.getCreateCPFPTableQuery(), await this.$checkIfTableExists('cpfp_clusters'));
await this.$executeQuery(this.getCreateTransactionsTableQuery(), await this.$checkIfTableExists('transactions'));
}
}
if (databaseSchemaVersion < 48 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0');
await this.$executeQuery(this.getCreateChannelsForensicsTableQuery(), await this.$checkIfTableExists('channels_forensics'));
}
}
/**
* Special case here for the `statistics` table - It appeared that somehow some dbs already had the `added` field indexed
@ -759,6 +764,25 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateChannelsForensicsTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS channels_forensics (
channel_id bigint(11) unsigned NOT NULL,
closing_fee bigint(20) unsigned DEFAULT 0,
node1_funding_balance bigint(20) unsigned DEFAULT 0,
node2_funding_balance bigint(20) unsigned DEFAULT 0,
node1_closing_balance bigint(20) unsigned DEFAULT 0,
node2_closing_balance bigint(20) unsigned DEFAULT 0,
funding_ratio float unsigned DEFAULT NULL,
closed_by varchar(66) DEFAULT NULL,
single_funded tinyint(1) default 0,
outputs JSON NOT NULL,
PRIMARY KEY (channel_id),
FOREIGN KEY (channel_id)
REFERENCES channels (id)
ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateNodesStatsQuery(): string {
return `CREATE TABLE IF NOT EXISTS node_stats (
id int(11) unsigned NOT NULL AUTO_INCREMENT,

View File

@ -128,6 +128,22 @@ class ChannelsApi {
}
}
public async $getChannelsWithoutSourceChecked(): Promise<any[]> {
try {
const query = `
SELECT channels.*, forensics.*
FROM channels
LEFT JOIN channels_forensics AS forensics ON forensics.channel_id = channels.id
WHERE channels.source_checked != 1
`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
try {
const query = `SELECT * FROM channels WHERE created IS NULL`;
@ -145,12 +161,16 @@ class ChannelsApi {
SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude,
n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude,
channels.*,
ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right
ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right,
forensics.closing_fee as closing_fee, forensics.node1_funding_balance as node1_funding_balance, forensics.node2_funding_balance as node2_funding_balance,
forensics.funding_ratio as funding_ratio, forensics.node1_closing_balance as node1_closing_balance, forensics.node2_closing_balance as node2_closing_balance,
forensics.closed_by as closed_by, forensics.single_funded as single_funded
FROM channels
LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key
LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key
LEFT JOIN channels_forensics AS forensics ON forensics.channel_id = channels.id
WHERE (
ns1.id = (
SELECT MAX(id)
@ -257,6 +277,118 @@ class ChannelsApi {
}
}
public async $getChannelForensicsByTransactionId(transactionId: string): Promise<any> {
try {
const query = `
SELECT
channels.id, channels.node1_public_key, channels.node2_public_key,
channels.closing_reason, channels.closing_transaction_id,
forensics.*
FROM channels
LEFT JOIN channels_forensics as forensics ON forensics.channel_id = channels.id
WHERE channels.closing_transaction_id = ?
`;
const [rows]: any = await DB.query(query, [transactionId]);
if (rows.length > 0) {
rows[0].outputs = JSON.parse(rows[0].outputs);
return rows[0];
}
} catch (e) {
logger.err('$getChannelForensicsByTransactionId error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise<void> {
try {
const query = `
INSERT INTO channels_forensics
(
channel_id,
node1_closing_balance,
node2_closing_balance,
closed_by,
closing_fee,
outputs
)
VALUES (?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
node1_closing_balance = ?,
node2_closing_balance = ?,
closed_by = ?,
closing_fee = ?,
outputs = ?
`;
const jsonOutputs = JSON.stringify(channelInfo.outputs);
await DB.query<ResultSetHeader>(query, [
channelInfo.id,
channelInfo.node1_closing_balance,
channelInfo.node2_closing_balance,
channelInfo.closed_by,
channelInfo.closing_fee,
jsonOutputs,
channelInfo.node1_closing_balance,
channelInfo.node2_closing_balance,
channelInfo.closed_by,
channelInfo.closing_fee,
jsonOutputs
]);
} catch (e) {
logger.err('$updateClosingInfo error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise<void> {
try {
const query = `
INSERT INTO channels_forensics
(
channel_id,
node1_funding_balance,
node2_funding_balance,
funding_ratio,
single_funded,
outputs
)
VALUES (?, ?, ?, ?, ?, 'null')
ON DUPLICATE KEY UPDATE
node1_funding_balance = ?,
node2_funding_balance = ?,
funding_ratio = ?,
single_funded = ?
`;
await DB.query<ResultSetHeader>(query, [
channelInfo.id,
channelInfo.node1_funding_balance,
channelInfo.node2_funding_balance,
channelInfo.funding_ratio,
channelInfo.single_funded ? 1 : 0,
channelInfo.node1_funding_balance,
channelInfo.node2_funding_balance,
channelInfo.funding_ratio,
channelInfo.single_funded ? 1 : 0,
]);
} catch (e) {
logger.err('$updateOpeningInfo error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $markChannelSourceChecked(id: string): Promise<void> {
try {
const query = `
UPDATE channels
SET source_checked = 1
WHERE id = ?
`;
await DB.query<ResultSetHeader>(query, [id]);
} catch (e) {
logger.err('$markChannelSourceChecked error: ' + (e instanceof Error ? e.message : e));
// don't throw - this data isn't essential
}
}
public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
try {
let channelStatusFilter;
@ -385,11 +517,15 @@ class ChannelsApi {
'transaction_id': channel.transaction_id,
'transaction_vout': channel.transaction_vout,
'closing_transaction_id': channel.closing_transaction_id,
'closing_fee': channel.closing_fee,
'closing_reason': channel.closing_reason,
'closing_date': channel.closing_date,
'updated_at': channel.updated_at,
'created': channel.created,
'status': channel.status,
'funding_ratio': channel.funding_ratio,
'closed_by': channel.closed_by,
'single_funded': !!channel.single_funded,
'node_left': {
'alias': channel.alias_left,
'public_key': channel.node1_public_key,
@ -404,6 +540,9 @@ class ChannelsApi {
'updated_at': channel.node1_updated_at,
'longitude': channel.node1_longitude,
'latitude': channel.node1_latitude,
'funding_balance': channel.node1_funding_balance,
'closing_balance': channel.node1_closing_balance,
'initiated_close': channel.closed_by === channel.node1_public_key ? true : undefined,
},
'node_right': {
'alias': channel.alias_right,
@ -419,6 +558,9 @@ class ChannelsApi {
'updated_at': channel.node2_updated_at,
'longitude': channel.node2_longitude,
'latitude': channel.node2_latitude,
'funding_balance': channel.node2_funding_balance,
'closing_balance': channel.node2_closing_balance,
'initiated_close': channel.closed_by === channel.node2_public_key ? true : undefined,
},
};
}

View File

@ -83,4 +83,10 @@ export namespace ILightningApi {
is_required: boolean;
is_known: boolean;
}
export interface ForensicOutput {
node?: 1 | 2;
type: number;
value: number;
}
}

View File

@ -304,6 +304,233 @@ class NetworkSyncService {
logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
}
}
private findLightningScript(vin: IEsploraApi.Vin): number {
const topElement = vin.witness[vin.witness.length - 2];
if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
// https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
if (topElement === '01') {
// top element is '01' to get in the revocation path
// 'Revoked Lightning Force Close';
// Penalty force closed
return 2;
} else {
// top element is '', this is a delayed to_local output
// 'Lightning Force Close';
return 3;
}
} else if (
/^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ||
/^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm)
) {
// https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
// https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
if (topElement.length === 66) {
// top element is a public key
// 'Revoked Lightning HTLC'; Penalty force closed
return 4;
} else if (topElement) {
// top element is a preimage
// 'Lightning HTLC';
return 5;
} else {
// top element is '' to get in the expiry of the script
// 'Expired Lightning HTLC';
return 6;
}
} else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) {
// https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors
if (topElement) {
// top element is a signature
// 'Lightning Anchor';
return 7;
} else {
// top element is '', it has been swept after 16 blocks
// 'Swept Lightning Anchor';
return 8;
}
}
return 1;
}
// If a channel open tx spends funds from a closed channel output,
// we can attribute that output to a specific counterparty
private async $runOpenedChannelsForensics(): Promise<void> {
let progress = 0;
try {
logger.info(`Started running open channel forensics...`);
const channels = await channelsApi.$getChannelsWithoutSourceChecked();
for (const openChannel of channels) {
const openTx = await bitcoinApi.$getRawTransaction(openChannel.transaction_id);
for (const input of openTx.vin) {
const closeChannel = await channelsApi.$getChannelForensicsByTransactionId(input.txid);
if (closeChannel) {
// this input directly spends a channel close output
await this.$attributeChannelBalances(closeChannel, openChannel, input);
} else {
// check if this input spends any swept channel close outputs
await this.$attributeSweptChannelCloses(openChannel, input);
}
}
// calculate how much of the total input value is attributable to the channel open output
openChannel.funding_ratio = openTx.vout[openChannel.transaction_vout].value / ((openTx.vout.reduce((sum, v) => sum + v.value, 0) || 1) + openTx.fee);
// save changes to the opening channel, and mark it as checked
if (openTx.vin.length === 1) {
openChannel.single_funded = true;
}
await channelsApi.$updateOpeningInfo(openChannel);
await channelsApi.$markChannelSourceChecked(openChannel.id);
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating channel opened channel forensics ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`Open channels forensics scan complete.`);
} catch (e) {
logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
}
}
// Check if a channel open tx input spends the result of a swept channel close output
private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise<void> {
const sweepTx = await bitcoinApi.$getRawTransaction(input.txid);
if (!sweepTx) {
logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`);
return;
}
const openContribution = sweepTx.vout[input.vout].value;
for (const sweepInput of sweepTx.vin) {
const lnScriptType = this.findLightningScript(sweepInput);
if (lnScriptType > 1) {
const closeChannel = await channelsApi.$getChannelForensicsByTransactionId(sweepInput.txid);
if (closeChannel) {
const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null);
await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator);
}
}
}
}
private async $attributeChannelBalances(
closeChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null,
initiator: 'remote' | 'local' | null = null
): Promise<void> {
// figure out which node controls the input/output
let openSide;
let closeLocal;
let closeRemote;
let matched = false;
let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart
if (openChannel.node1_public_key === closeChannel.node1_public_key) {
openSide = 1;
closeLocal = 1;
closeRemote = 2;
matched = true;
} else if (openChannel.node1_public_key === closeChannel.node2_public_key) {
openSide = 1;
closeLocal = 2;
closeRemote = 1;
matched = true;
}
if (openChannel.node2_public_key === closeChannel.node1_public_key) {
openSide = 2;
closeLocal = 1;
closeRemote = 2;
if (matched) {
ambiguous = true;
}
matched = true;
} else if (openChannel.node2_public_key === closeChannel.node2_public_key) {
openSide = 2;
closeLocal = 2;
closeRemote = 1;
if (matched) {
ambiguous = true;
}
matched = true;
}
if (matched && !ambiguous) {
// fetch closing channel transaction and perform forensics on the outputs
let closingTx: IEsploraApi.Transaction | undefined;
let outspends: IEsploraApi.Outspend[] | undefined;
try {
closingTx = await bitcoinApi.$getRawTransaction(input.txid);
outspends = await bitcoinApi.$getOutspends(input.txid);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + closeChannel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
}
if (!outspends || !closingTx) {
return;
}
if (!closeChannel.outputs) {
closeChannel.outputs = closeChannel.outputs || closingTx.vout.map(vout => {
return {
type: 0,
value: vout.value,
};
});
}
for (let i = 0; i < outspends.length; i++) {
const outspend = outspends[i];
const output = closeChannel.outputs[i];
if (outspend.spent && outspend.txid) {
try {
const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
if (spendingTx) {
output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
}
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
}
} else {
output.type = 0;
}
}
// attribute outputs to each counterparty, and sum up total known balances
closeChannel.outputs[input.vout].node = closeLocal;
const isPenalty = closeChannel.outputs.filter((out) => out.type === 2 || out.type === 4).length > 0;
const normalOutput = [1,3].includes(closeChannel.outputs[input.vout].type);
let localClosingBalance = 0;
let remoteClosingBalance = 0;
for (const output of closeChannel.outputs) {
if (isPenalty) {
// penalty close, so local node takes everything
localClosingBalance += output.value;
} else if (output.node) {
// this output determinstically linked to one of the counterparties
if (output.node === closeLocal) {
localClosingBalance += output.value;
} else {
remoteClosingBalance += output.value;
}
} else if (normalOutput && (output.type === 1 || output.type === 3)) {
// local node had one main output, therefore remote node takes the other
remoteClosingBalance += output.value;
}
}
openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || closingTx?.vout[input.vout]?.value || 0);
closeChannel[`node${closeLocal}_closing_balance`] = localClosingBalance;
closeChannel[`node${closeRemote}_closing_balance`] = remoteClosingBalance;
closeChannel.closing_fee = closingTx.fee;
if (initiator) {
const initiatorSide = initiator === 'remote' ? closeRemote : closeLocal;
closeChannel.closed_by = closeChannel[`node${initiatorSide}_public_key`];
}
// save changes to the closing channel
await channelsApi.$updateClosingInfo(closeChannel);
}
}
}
export default new NetworkSyncService();