From 9e4fe40ca396bf9305dad75de7d97285de688857 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sun, 19 Mar 2023 10:46:38 +0900 Subject: [PATCH 1/3] When a re-org happens, keep the block templates for audit --- backend/src/api/blocks.ts | 6 ++-- backend/src/repositories/BlocksRepository.ts | 29 ++----------------- .../repositories/BlocksSummariesRepository.ts | 14 +++++---- .../src/repositories/HashratesRepository.ts | 2 +- 4 files changed, 16 insertions(+), 35 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index eee5dae6e..dacf1bfd5 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -589,11 +589,11 @@ class Blocks { if (!fastForwarded) { const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1); if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) { - logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`); + logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`, logger.tags.mining); // We assume there won't be a reorg with more than 10 block depth await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10); await HashratesRepository.$deleteLastEntries(); - await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10); + await BlocksSummariesRepository.$deleteTransactionsFrom(lastBlock.height - 10); // Will be re-index in formatDbBlockIntoExtendedBlock() await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10); for (let i = 10; i >= 0; --i) { const newBlock = await this.$indexBlock(lastBlock.height - i); @@ -604,7 +604,7 @@ class Blocks { } await mining.$indexDifficultyAdjustments(); await DifficultyAdjustmentsRepository.$deleteLastAdjustment(); - logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`); + logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining); indexer.reindex(); } await blocksRepository.$saveBlockInDatabase(blockExtended); diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 4758c0708..8d87c9537 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -466,30 +466,6 @@ class BlocksRepository { } } - /** - * Get one block by hash - */ - public async $getBlockByHash(hash: string): Promise { - try { - const query = ` - SELECT ${BLOCK_DB_FIELDS} - FROM blocks - JOIN pools ON blocks.pool_id = pools.id - WHERE hash = ?; - `; - const [rows]: any[] = await DB.query(query, [hash]); - - if (rows.length <= 0) { - return null; - } - - return await this.formatDbBlockIntoExtendedBlock(rows[0]); - } catch (e) { - logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e)); - throw e; - } - } - /** * Return blocks difficulty */ @@ -599,7 +575,7 @@ class BlocksRepository { if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) { logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`); await this.$deleteBlocksFrom(blocks[idx - 1].height); - await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height); + await BlocksSummariesRepository.$deleteTransactionsFrom(blocks[idx - 1].height); await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800); await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height); return false; @@ -619,7 +595,7 @@ class BlocksRepository { * Delete blocks from the database from blockHeight */ public async $deleteBlocksFrom(blockHeight: number) { - logger.info(`Delete newer blocks from height ${blockHeight} from the database`); + logger.info(`Delete newer blocks from height ${blockHeight} from the database`, logger.tags.mining); try { await DB.query(`DELETE FROM blocks where height >= ${blockHeight}`); @@ -997,6 +973,7 @@ class BlocksRepository { } // If we're missing block summary related field, check if we can populate them on the fly now + // This is for example triggered upon re-org if (Common.blocksSummariesIndexingEnabled() && (extras.medianFeeAmt === null || extras.feePercentiles === null)) { diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index 2724ddcf5..b583a5f2c 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -17,7 +17,7 @@ class BlocksSummariesRepository { return undefined; } - public async $saveSummary(params: { height: number, mined?: BlockSummary}) { + public async $saveSummary(params: { height: number, mined?: BlockSummary}): Promise { const blockId = params.mined?.id; try { const transactions = JSON.stringify(params.mined?.transactions || []); @@ -71,13 +71,17 @@ class BlocksSummariesRepository { /** * Delete blocks from the database from blockHeight */ - public async $deleteBlocksFrom(blockHeight: number) { - logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`); + public async $deleteTransactionsFrom(blockHeight: number): Promise { + logger.info(`Delete blocks summaries transactions from height ${blockHeight} from the database, but keep templates`, logger.tags.mining); try { - await DB.query(`DELETE FROM blocks_summaries where height >= ${blockHeight}`); + await DB.query(` + UPDATE blocks_summaries + SET transactions = '[]' + WHERE height >= ${blockHeight} + `); } catch (e) { - logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot delete blocks summaries transactions. Reason: ' + (e instanceof Error ? e.message : e)); } } diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 875f77b34..96cbf6f75 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -220,7 +220,7 @@ class HashratesRepository { * Delete hashrates from the database from timestamp */ public async $deleteHashratesFromTimestamp(timestamp: number) { - logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`); + logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`, logger.tags.mining); try { await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]); From 44bbb472d3683bd83efba7d50126d08ced718720 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 31 Mar 2023 12:08:05 +0900 Subject: [PATCH 2/3] Keep re-org'ed block summaries in the database --- backend/src/api/blocks.ts | 5 ++--- backend/src/repositories/BlocksRepository.ts | 2 +- .../repositories/BlocksSummariesRepository.ts | 16 +++++++++++++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index dacf1bfd5..cc7dc6588 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -593,7 +593,6 @@ class Blocks { // We assume there won't be a reorg with more than 10 block depth await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10); await HashratesRepository.$deleteLastEntries(); - await BlocksSummariesRepository.$deleteTransactionsFrom(lastBlock.height - 10); // Will be re-index in formatDbBlockIntoExtendedBlock() await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10); for (let i = 10; i >= 0; --i) { const newBlock = await this.$indexBlock(lastBlock.height - i); @@ -736,7 +735,7 @@ class Blocks { // Index the response if needed if (Common.blocksSummariesIndexingEnabled() === true) { - await BlocksSummariesRepository.$saveSummary({height: block.height, mined: summary}); + await BlocksSummariesRepository.$saveTransactions(block.height, block.hash, summary.transactions); } return summary.transactions; @@ -852,7 +851,7 @@ class Blocks { if (cleanBlock.fee_amt_percentiles === null) { const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); const summary = this.summarizeBlock(block); - await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary }); + await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions); cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); } if (cleanBlock.fee_amt_percentiles !== null) { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 8d87c9537..251d86403 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -981,7 +981,7 @@ class BlocksRepository { if (extras.feePercentiles === null) { const block = await bitcoinClient.getBlock(dbBlk.id, 2); const summary = blocks.summarizeBlock(block); - await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary }); + await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.hash, summary.transactions); extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); } if (extras.feePercentiles !== null) { diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index b583a5f2c..570aee473 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -1,6 +1,6 @@ import DB from '../database'; import logger from '../logger'; -import { BlockSummary } from '../mempool.interfaces'; +import { BlockSummary, TransactionStripped } from '../mempool.interfaces'; class BlocksSummariesRepository { public async $getByBlockId(id: string): Promise { @@ -37,6 +37,20 @@ class BlocksSummariesRepository { } } + public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionStripped[]): Promise { + try { + const transactionsStr = JSON.stringify(transactions); + await DB.query(` + INSERT INTO blocks_summaries + SET height = ?, transactions = ?, id = ? + ON DUPLICATE KEY UPDATE transactions = ?`, + [blockHeight, transactionsStr, blockId, transactionsStr]); + } catch (e: any) { + logger.debug(`Cannot save block summary transactions for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + public async $saveTemplate(params: { height: number, template: BlockSummary}) { const blockId = params.template?.id; try { From 816fb3bf01fcaff65666dd5c4ed1bfea49a1d6da Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 31 Mar 2023 12:22:26 +0900 Subject: [PATCH 3/3] Don't delete transactions when checking if the current chain is valid --- backend/src/repositories/BlocksRepository.ts | 1 - .../repositories/BlocksSummariesRepository.ts | 17 ----------------- 2 files changed, 18 deletions(-) diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 251d86403..69d597e1f 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -575,7 +575,6 @@ class BlocksRepository { if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) { logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`); await this.$deleteBlocksFrom(blocks[idx - 1].height); - await BlocksSummariesRepository.$deleteTransactionsFrom(blocks[idx - 1].height); await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800); await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height); return false; diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index 570aee473..de70322bd 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -82,23 +82,6 @@ class BlocksSummariesRepository { return []; } - /** - * Delete blocks from the database from blockHeight - */ - public async $deleteTransactionsFrom(blockHeight: number): Promise { - logger.info(`Delete blocks summaries transactions from height ${blockHeight} from the database, but keep templates`, logger.tags.mining); - - try { - await DB.query(` - UPDATE blocks_summaries - SET transactions = '[]' - WHERE height >= ${blockHeight} - `); - } catch (e) { - logger.err('Cannot delete blocks summaries transactions. Reason: ' + (e instanceof Error ? e.message : e)); - } - } - /** * Get the fee percentiles if the block has already been indexed, [] otherwise *