From 5bee54a2bf9f5dfdd56ddf3cc7799329319b369e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 17 Aug 2023 02:42:59 +0900 Subject: [PATCH] Use new bulk endpoints to speed up forensics --- .../bitcoin/bitcoin-api-abstract-factory.ts | 1 + backend/src/api/bitcoin/bitcoin-api.ts | 17 +- backend/src/api/bitcoin/esplora-api.ts | 13 ++ .../src/tasks/lightning/forensics.service.ts | 193 ++++++++++++------ 4 files changed, 155 insertions(+), 69 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index a76b93e8d..6f20dad92 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -24,6 +24,7 @@ export interface AbstractBitcoinApi { $getOutspend(txId: string, vout: number): Promise; $getOutspends(txId: string): Promise; $getBatchedOutspends(txId: string[]): Promise; + $getBatchedOutspendsInternal(txId: string[]): Promise; startHealthChecks(): void; } diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 1be7993b8..9e4cbdd8b 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -60,8 +60,17 @@ class BitcoinApi implements AbstractBitcoinApi { }); } - $getRawTransactions(txids: string[]): Promise { - throw new Error('Method getRawTransactions not supported by the Bitcoin RPC API.'); + async $getRawTransactions(txids: string[]): Promise { + const txs: IEsploraApi.Transaction[] = []; + for (const txid of txids) { + try { + const tx = await this.$getRawTransaction(txid, false, true); + txs.push(tx); + } catch (err) { + // skip failures + } + } + return txs; } $getMempoolTransactions(txids: string[]): Promise { @@ -202,6 +211,10 @@ class BitcoinApi implements AbstractBitcoinApi { return outspends; } + async $getBatchedOutspendsInternal(txId: string[]): Promise { + return this.$getBatchedOutspends(txId); + } + $getEstimatedHashrate(blockHeight: number): Promise { // 120 is the default block span in Core return this.bitcoindClient.getNetworkHashPs(120, blockHeight); diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index d400dad7a..574113ae6 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -301,6 +301,19 @@ class ElectrsApi implements AbstractBitcoinApi { throw new Error('Method not implemented.'); } + async $getBatchedOutspendsInternal(txids: string[]): Promise { + const allOutspends: IEsploraApi.Outspend[][] = []; + const sliceLength = 50; + for (let i = 0; i < Math.ceil(txids.length / sliceLength); i++) { + const slice = txids.slice(i * sliceLength, (i + 1) * sliceLength); + const sliceOutspends = await this.failoverRouter.$get('/txs/outspends', 'json', { txids: slice.join(',') }); + for (const outspends of sliceOutspends) { + allOutspends.push(outspends); + } + } + return allOutspends; + } + public startHealthChecks(): void { this.failoverRouter.startHealthChecks(); } diff --git a/backend/src/tasks/lightning/forensics.service.ts b/backend/src/tasks/lightning/forensics.service.ts index 65ea61dc1..8a9bb825f 100644 --- a/backend/src/tasks/lightning/forensics.service.ts +++ b/backend/src/tasks/lightning/forensics.service.ts @@ -15,8 +15,6 @@ class ForensicsService { txCache: { [txid: string]: IEsploraApi.Transaction } = {}; tempCached: string[] = []; - constructor() {} - public async $startService(): Promise { logger.info('Starting lightning network forensics service'); @@ -66,93 +64,154 @@ class ForensicsService { */ public async $runClosedChannelsForensics(onlyNewChannels: boolean = false): Promise { + // Only Esplora backend can retrieve spent transaction outputs if (config.MEMPOOL.BACKEND !== 'esplora') { return; } - let progress = 0; - try { logger.debug(`Started running closed channel forensics...`); - let channels; + let remainingChannels; if (onlyNewChannels) { - channels = await channelsApi.$getClosedChannelsWithoutReason(); + remainingChannels = await channelsApi.$getClosedChannelsWithoutReason(); } else { - channels = await channelsApi.$getUnresolvedClosedChannels(); + remainingChannels = await channelsApi.$getUnresolvedClosedChannels(); } - for (const channel of channels) { - let reason = 0; - let resolvedForceClose = false; - // Only Esplora backend can retrieve spent transaction outputs - const cached: string[] = []; + let progress = 0; + const sliceLength = 1000; + // process batches of 1000 channels + for (let i = 0; i < Math.ceil(remainingChannels.length / sliceLength); i++) { + const channels = remainingChannels.slice(i * sliceLength, (i + 1) * sliceLength); + + let allOutspends: IEsploraApi.Outspend[][] = []; + const forceClosedChannels: { channel: any, cachedSpends: string[] }[] = []; + + // fetch outspends in bulk try { - let outspends: IEsploraApi.Outspend[] | undefined; - try { - outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id); - await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT); - } catch (e) { - logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); - continue; - } - const lightningScriptReasons: number[] = []; - for (const outspend of outspends) { - if (outspend.spent && outspend.txid) { - let spendingTx = await this.fetchTransaction(outspend.txid); - if (!spendingTx) { - continue; + const outspendTxids = channels.map(channel => channel.closing_transaction_id); + allOutspends = await bitcoinApi.$getBatchedOutspendsInternal(outspendTxids); + logger.info(`Fetched outspends for ${allOutspends.length} txs from esplora for lightning forensics`); + await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT); + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/txs/outspends'}. Reason ${e instanceof Error ? e.message : e}`); + } + // fetch spending transactions in bulk and load into txCache + try { + const newSpendingTxids: { [txid: string]: boolean } = {}; + for (const outspends of allOutspends) { + for (const outspend of outspends) { + if (outspend.spent && outspend.txid) { + if (!this.txCache[outspend.txid]) { + newSpendingTxids[outspend.txid] = true; + } } - cached.push(spendingTx.txid); - const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]); - lightningScriptReasons.push(lightningScript); } } - const filteredReasons = lightningScriptReasons.filter((r) => r !== 1); - if (filteredReasons.length) { - if (filteredReasons.some((r) => r === 2 || r === 4)) { - reason = 3; - } else { - reason = 2; - resolvedForceClose = true; - } - } else { - /* - We can detect a commitment transaction (force close) by reading Sequence and Locktime - https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction - */ - let closingTx = await this.fetchTransaction(channel.closing_transaction_id, true); - if (!closingTx) { + const allOutspendTxs = await bitcoinApi.$getRawTransactions(Object.keys(newSpendingTxids)); + logger.info(`Fetched ${allOutspendTxs.length} out-spending txs from esplora for lightning forensics`); + for (const tx of allOutspendTxs) { + this.txCache[tx.txid] = tx; + } + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/txs'}. Reason ${e instanceof Error ? e.message : e}`); + } + + // process each outspend + for (const [index, channel] of channels.entries()) { + let reason = 0; + const cached: string[] = []; + try { + const outspends = allOutspends[index]; + if (!outspends || !outspends.length) { + // outspends are missing continue; } - cached.push(closingTx.txid); - const sequenceHex: string = closingTx.vin[0].sequence.toString(16); - const locktimeHex: string = closingTx.locktime.toString(16); - if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') { - reason = 2; // Here we can't be sure if it's a penalty or not - } else { - reason = 1; + const lightningScriptReasons: number[] = []; + for (const outspend of outspends) { + if (outspend.spent && outspend.txid) { + const spendingTx = this.txCache[outspend.txid]; + if (!spendingTx) { + continue; + } + cached.push(spendingTx.txid); + const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]); + lightningScriptReasons.push(lightningScript); + } } - } - if (reason) { - logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.'); - await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]); - if (reason === 2 && resolvedForceClose) { - await DB.query(`UPDATE channels SET closing_resolved = ? WHERE id = ?`, [true, channel.id]); - } - if (reason !== 2 || resolvedForceClose) { + const filteredReasons = lightningScriptReasons.filter((r) => r !== 1); + if (filteredReasons.length) { + if (filteredReasons.some((r) => r === 2 || r === 4)) { + // Force closed with penalty + reason = 3; + } else { + // Force closed without penalty + reason = 2; + await DB.query(`UPDATE channels SET closing_resolved = ? WHERE id = ?`, [true, channel.id]); + } + await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]); + // clean up cached transactions cached.forEach(txid => { delete this.txCache[txid]; }); + } else { + forceClosedChannels.push({ channel, cachedSpends: cached }); } + } catch (e) { + logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`); } - } catch (e) { - logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`); } - ++progress; + // fetch force-closing transactions in bulk + try { + const newClosingTxids: { [txid: string]: boolean } = {}; + for (const { channel } of forceClosedChannels) { + if (!this.txCache[channel.closing_transaction_id]) { + newClosingTxids[channel.closing_transaction_id] = true; + } + } + const closingTxs = await bitcoinApi.$getRawTransactions(Object.keys(newClosingTxids)); + logger.info(`Fetched ${closingTxs.length} closing txs from esplora for lightning forensics`); + for (const tx of closingTxs) { + this.txCache[tx.txid] = tx; + } + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/txs'}. Reason ${e instanceof Error ? e.message : e}`); + } + + // process channels with no lightning script reasons + for (const { channel, cachedSpends } of forceClosedChannels) { + const closingTx = this.txCache[channel.closing_transaction_id]; + if (!closingTx) { + // no channel close transaction found yet + continue; + } + /* + We can detect a commitment transaction (force close) by reading Sequence and Locktime + https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction + */ + const sequenceHex: string = closingTx.vin[0].sequence.toString(16); + const locktimeHex: string = closingTx.locktime.toString(16); + let reason; + if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') { + // Force closed, but we can't be sure if it's a penalty or not + reason = 2; + } else { + // Mutually closed + reason = 1; + // clean up cached transactions + delete this.txCache[closingTx.txid]; + for (const txid of cachedSpends) { + delete this.txCache[txid]; + } + } + await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]); + } + + progress += channels.length; const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); if (elapsedSeconds > 10) { - logger.debug(`Updating channel closed channel forensics ${progress}/${channels.length}`); + logger.debug(`Updating channel closed channel forensics ${progress}/${remainingChannels.length}`); this.loggerTimer = new Date().getTime() / 1000; } } @@ -221,7 +280,7 @@ class ForensicsService { const channels = await channelsApi.$getChannelsWithoutSourceChecked(); for (const openChannel of channels) { - let openTx = await this.fetchTransaction(openChannel.transaction_id, true); + const openTx = await this.fetchTransaction(openChannel.transaction_id, true); if (!openTx) { continue; } @@ -276,7 +335,7 @@ class ForensicsService { // Check if a channel open tx input spends the result of a swept channel close output private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise { - let sweepTx = await this.fetchTransaction(input.txid, true); + const sweepTx = await this.fetchTransaction(input.txid, true); if (!sweepTx) { logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`); return; @@ -335,7 +394,7 @@ class ForensicsService { if (matched && !ambiguous) { // fetch closing channel transaction and perform forensics on the outputs - let prevChannelTx = await this.fetchTransaction(input.txid, true); + const prevChannelTx = await this.fetchTransaction(input.txid, true); let outspends: IEsploraApi.Outspend[] | undefined; try { outspends = await bitcoinApi.$getOutspends(input.txid); @@ -430,7 +489,7 @@ class ForensicsService { } await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT); } catch (e) { - logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid}. Reason ${e instanceof Error ? e.message : e}`); return null; } }