From db8ba7c938ff3118b54f847ba2da2a905e406c66 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 30 Jan 2024 16:41:35 +0000 Subject: [PATCH] More robust error handling and logging during summaries indexing --- backend/src/api/blocks.ts | 37 +++++++++++++------ backend/src/api/common.ts | 13 ++++++- .../repositories/BlocksAuditsRepository.ts | 2 +- backend/src/repositories/BlocksRepository.ts | 14 +++---- .../repositories/BlocksSummariesRepository.ts | 3 +- backend/src/utils/secp256k1.ts | 3 ++ 6 files changed, 49 insertions(+), 23 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 2cd043fe2..837bc0ee9 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,7 +2,7 @@ import config from '../config'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; @@ -451,7 +451,9 @@ class Blocks { if (config.MEMPOOL.BACKEND === 'esplora') { const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx)); const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs); - await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary + if (cpfpSummary) { + await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary + } } else { await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary } @@ -995,11 +997,11 @@ class Blocks { return state; } - private updateTimerProgress(state, msg) { + private updateTimerProgress(state, msg): void { state.progress = msg; } - private clearTimer(state) { + private clearTimer(state): void { if (state.timer) { clearTimeout(state.timer); } @@ -1088,13 +1090,19 @@ class Blocks { summary = { id: hash, transactions: cpfpSummary.transactions.map(tx => { + let flags: number = 0; + try { + flags = tx.flags || Common.getTransactionFlags(tx); + } catch (e) { + logger.warn('Failed to classify transaction: ' + (e instanceof Error ? e.message : e)); + } return { txid: tx.txid, fee: tx.fee || 0, vsize: tx.vsize, value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)), rate: tx.effectiveFeePerVsize, - flags: tx.flags || Common.getTransactionFlags(tx), + flags: flags, }; }), }; @@ -1284,7 +1292,7 @@ class Blocks { return blocks; } - public async $getBlockAuditSummary(hash: string): Promise { + public async $getBlockAuditSummary(hash: string): Promise { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { return BlocksAuditsRepository.$getBlockAudit(hash); } else { @@ -1304,7 +1312,7 @@ class Blocks { return this.currentBlockHeight; } - public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise { + public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise { let transactions = txs; if (!transactions) { if (config.MEMPOOL.BACKEND === 'esplora') { @@ -1319,14 +1327,19 @@ class Blocks { } } - const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]); + if (transactions?.length != null) { + const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]); - await this.$saveCpfp(hash, height, summary); + await this.$saveCpfp(hash, height, summary); - const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); - await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); + const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); + await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); - return summary; + return summary; + } else { + logger.err(`Cannot index CPFP for block ${height} - missing transaction data`); + return null; + } } public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise { diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 208c67d70..63c215a8f 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -6,6 +6,7 @@ import { NodeSocket } from '../repositories/NodesSocketsRepository'; import { isIP } from 'net'; import transactionUtils from './transaction-utils'; import { isPoint } from '../utils/secp256k1'; +import logger from '../logger'; export class Common { static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ? '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49' @@ -261,6 +262,9 @@ export class Common { case 'v0_p2wpkh': flags |= TransactionFlags.p2wpkh; break; case 'v0_p2wsh': flags |= TransactionFlags.p2wsh; break; case 'v1_p2tr': { + if (!vin.witness?.length) { + throw new Error('Taproot input missing witness data'); + } flags |= TransactionFlags.p2tr; // in taproot, if the last witness item begins with 0x50, it's an annex const hasAnnex = vin.witness?.[vin.witness.length - 1].startsWith('50'); @@ -301,7 +305,7 @@ export class Common { case 'p2pk': { flags |= TransactionFlags.p2pk; // detect fake pubkey (i.e. not a valid DER point on the secp256k1 curve) - hasFakePubkey = hasFakePubkey || !isPoint(vout.scriptpubkey.slice(2, -2)); + hasFakePubkey = hasFakePubkey || !isPoint(vout.scriptpubkey?.slice(2, -2)); } break; case 'multisig': { flags |= TransactionFlags.p2ms; @@ -348,7 +352,12 @@ export class Common { } static classifyTransaction(tx: TransactionExtended): TransactionClassified { - const flags = Common.getTransactionFlags(tx); + let flags = 0; + try { + flags = Common.getTransactionFlags(tx); + } catch (e) { + logger.warn('Failed to add classification flags to transaction: ' + (e instanceof Error ? e.message : e)); + } tx.flags = flags; return { ...Common.stripTransaction(tx), diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index c17958d2b..585abdedf 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -59,7 +59,7 @@ class BlocksAuditRepositories { } } - public async $getBlockAudit(hash: string): Promise { + public async $getBlockAudit(hash: string): Promise { try { const [rows]: any[] = await DB.query( `SELECT blocks_audits.height, blocks_audits.hash as id, UNIX_TIMESTAMP(blocks_audits.time) as timestamp, diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index a2a084265..18346740b 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -5,7 +5,7 @@ import logger from '../logger'; import { Common } from '../api/common'; import PoolsRepository from './PoolsRepository'; import HashratesRepository from './HashratesRepository'; -import { escape } from 'mysql2'; +import { RowDataPacket, escape } from 'mysql2'; import BlocksSummariesRepository from './BlocksSummariesRepository'; import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository'; import bitcoinClient from '../api/bitcoin/bitcoin-client'; @@ -802,10 +802,10 @@ class BlocksRepository { /** * Get a list of blocks that have been indexed */ - public async $getIndexedBlocks(): Promise { + public async $getIndexedBlocks(): Promise<{ height: number, hash: string }[]> { try { - const [rows]: any = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`); - return rows; + const [rows] = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`) as RowDataPacket[][]; + return rows as { height: number, hash: string }[]; } catch (e) { logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e)); throw e; @@ -815,7 +815,7 @@ class BlocksRepository { /** * Get a list of blocks that have not had CPFP data indexed */ - public async $getCPFPUnindexedBlocks(): Promise { + public async $getCPFPUnindexedBlocks(): Promise { try { const blockchainInfo = await bitcoinClient.getBlockchainInfo(); const currentBlockHeight = blockchainInfo.blocks; @@ -825,13 +825,13 @@ class BlocksRepository { } const minHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1); - const [rows]: any[] = await DB.query(` + const [rows] = await DB.query(` SELECT height FROM compact_cpfp_clusters WHERE height <= ? AND height >= ? GROUP BY height ORDER BY height DESC; - `, [currentBlockHeight, minHeight]); + `, [currentBlockHeight, minHeight]) as RowDataPacket[][]; const indexedHeights = {}; rows.forEach((row) => { indexedHeights[row.height] = true; }); diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index f85914e31..63ad5ddf2 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -1,3 +1,4 @@ +import { RowDataPacket } from 'mysql2'; import DB from '../database'; import logger from '../logger'; import { BlockSummary, TransactionClassified } from '../mempool.interfaces'; @@ -69,7 +70,7 @@ class BlocksSummariesRepository { public async $getIndexedSummariesId(): Promise { try { - const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`); + const [rows] = await DB.query(`SELECT id from blocks_summaries`) as RowDataPacket[][]; return rows.map(row => row.id); } catch (e) { logger.err(`Cannot get block summaries id list. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/utils/secp256k1.ts b/backend/src/utils/secp256k1.ts index cc731f17d..9e0f6dc3b 100644 --- a/backend/src/utils/secp256k1.ts +++ b/backend/src/utils/secp256k1.ts @@ -31,6 +31,9 @@ const curveP = BigInt(`0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF * @returns {boolean} true if the point is on the SECP256K1 curve */ export function isPoint(pointHex: string): boolean { + if (!pointHex?.length) { + return false; + } if ( !( // is uncompressed