From 5792dee553f03c632fc5888a35490a27e772b281 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 27 Feb 2023 10:47:04 +0900 Subject: [PATCH 1/5] Use bitcoinApiFactory when we don't need verbose blocks or confirmation number --- backend/src/api/blocks.ts | 14 ++++++-------- backend/src/api/chain-tips.ts | 4 ++-- backend/src/api/mining/mining.ts | 14 ++++++++------ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 204419496..068184949 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -479,7 +479,7 @@ class Blocks { loadingIndicators.setProgress('block-indexing', progress, false); } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); - const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -527,13 +527,13 @@ class Blocks { if (blockchainInfo.blocks === blockchainInfo.headers) { const heightDiff = blockHeightTip % 2016; const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); - const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); this.lastDifficultyAdjustmentTime = block.timestamp; this.currentDifficulty = block.difficulty; if (blockHeightTip >= 2016) { const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); - const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash) + const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash); this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; logger.debug(`Initial difficulty adjustment data set.`); } @@ -657,7 +657,7 @@ class Blocks { } const blockHash = await bitcoinApi.$getBlockHash(height); - const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -681,7 +681,7 @@ class Blocks { // Block has already been indexed if (Common.indexingEnabled()) { const dbBlock = await blocksRepository.$getBlockByHash(hash); - if (dbBlock != null) { + if (dbBlock !== null) { return prepareBlock(dbBlock); } } @@ -691,10 +691,8 @@ class Blocks { return await bitcoinApi.$getBlock(hash); } - let block = await bitcoinClient.getBlock(hash); - block = prepareBlock(block); - // Bitcoin network, add our custom data on top + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); const transactions = await this.$getTransactionsExtended(hash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); if (Common.indexingEnabled()) { diff --git a/backend/src/api/chain-tips.ts b/backend/src/api/chain-tips.ts index 3384ebb19..46a06f703 100644 --- a/backend/src/api/chain-tips.ts +++ b/backend/src/api/chain-tips.ts @@ -1,5 +1,5 @@ -import logger from "../logger"; -import bitcoinClient from "./bitcoin/bitcoin-client"; +import logger from '../logger'; +import bitcoinClient from './bitcoin/bitcoin-client'; export interface ChainTip { height: number; diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index f33a68dcb..e9b569485 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -11,6 +11,8 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust import config from '../../config'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import PricesRepository from '../../repositories/PricesRepository'; +import bitcoinApiFactory from '../bitcoin/bitcoin-api-factory'; +import { IEsploraApi } from '../bitcoin/esplora-api.interface'; class Mining { blocksPriceIndexingRunning = false; @@ -189,8 +191,8 @@ class Mining { try { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; - const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); - const genesisTimestamp = genesisBlock.time * 1000; + const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); const hashrates: any[] = []; @@ -292,8 +294,8 @@ class Mining { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; try { - const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); - const genesisTimestamp = genesisBlock.time * 1000; + const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); const lastMidnight = this.getDateMidnight(new Date()); let toTimestamp = Math.round(lastMidnight.getTime()); @@ -394,13 +396,13 @@ class Mining { } const blocks: any = await BlocksRepository.$getBlocksDifficulty(); - const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0)); let currentDifficulty = genesisBlock.difficulty; let totalIndexed = 0; if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) { await DifficultyAdjustmentsRepository.$saveAdjustments({ - time: genesisBlock.time, + time: genesisBlock.timestamp, height: 0, difficulty: currentDifficulty, adjustment: 0.0, From 0aff276a5c8d9eb5d1f8688836d95a0427e4bb73 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 27 Feb 2023 18:00:00 +0900 Subject: [PATCH 2/5] Enforce BlockExtended use for block indexing - Unify /api/v1/block(s) API(s) response format --- .../src/api/bitcoin/bitcoin-api.interface.ts | 31 +++ backend/src/api/bitcoin/bitcoin-api.ts | 2 +- .../src/api/bitcoin/esplora-api.interface.ts | 2 +- backend/src/api/blocks.ts | 224 +++++++++--------- backend/src/api/chain-tips.ts | 6 +- backend/src/api/pools-parser.ts | 2 + backend/src/index.ts | 9 +- backend/src/mempool.interfaces.ts | 69 +++--- backend/src/repositories/BlocksRepository.ts | 206 +++++++++++----- backend/src/repositories/PoolsRepository.ts | 6 +- backend/src/utils/blocks-utils.ts | 33 --- .../src/app/interfaces/node-api.interface.ts | 1 - 12 files changed, 352 insertions(+), 239 deletions(-) delete mode 100644 backend/src/utils/blocks-utils.ts diff --git a/backend/src/api/bitcoin/bitcoin-api.interface.ts b/backend/src/api/bitcoin/bitcoin-api.interface.ts index 54d666794..3afc22897 100644 --- a/backend/src/api/bitcoin/bitcoin-api.interface.ts +++ b/backend/src/api/bitcoin/bitcoin-api.interface.ts @@ -172,4 +172,35 @@ export namespace IBitcoinApi { } } + export interface BlockStats { + "avgfee": number; + "avgfeerate": number; + "avgtxsize": number; + "blockhash": string; + "feerate_percentiles": [number, number, number, number, number]; + "height": number; + "ins": number; + "maxfee": number; + "maxfeerate": number; + "maxtxsize": number; + "medianfee": number; + "mediantime": number; + "mediantxsize": number; + "minfee": number; + "minfeerate": number; + "mintxsize": number; + "outs": number; + "subsidy": number; + "swtotal_size": number; + "swtotal_weight": number; + "swtxs": number; + "time": number; + "total_out": number; + "total_size": number; + "total_weight": number; + "totalfee": number; + "txs": number; + "utxo_increase": number; + "utxo_size_inc": number; + } } diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 117245ef8..e20fe9e34 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -28,7 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi { size: block.size, weight: block.weight, previousblockhash: block.previousblockhash, - medianTime: block.mediantime, + mediantime: block.mediantime, }; } diff --git a/backend/src/api/bitcoin/esplora-api.interface.ts b/backend/src/api/bitcoin/esplora-api.interface.ts index eaf6476f4..6d50bddfd 100644 --- a/backend/src/api/bitcoin/esplora-api.interface.ts +++ b/backend/src/api/bitcoin/esplora-api.interface.ts @@ -88,7 +88,7 @@ export namespace IEsploraApi { size: number; weight: number; previousblockhash: string; - medianTime?: number; + mediantime: number; } export interface Address { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 068184949..75d1ec300 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,7 +2,7 @@ import config from '../config'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; @@ -13,7 +13,6 @@ import poolsRepository from '../repositories/PoolsRepository'; import blocksRepository from '../repositories/BlocksRepository'; import loadingIndicators from './loading-indicators'; import BitcoinApi from './bitcoin/bitcoin-api'; -import { prepareBlock } from '../utils/blocks-utils'; import BlocksRepository from '../repositories/BlocksRepository'; import HashratesRepository from '../repositories/HashratesRepository'; import indexer from '../indexer'; @@ -143,7 +142,7 @@ class Blocks { * @param block * @returns BlockSummary */ - private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary { + public summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary { const stripped = block.tx.map((tx) => { return { txid: tx.txid, @@ -166,80 +165,81 @@ class Blocks { * @returns BlockExtended */ private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { - const blk: BlockExtended = Object.assign({ extras: {} }, block); - blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); - blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); - blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig; - blk.extras.usd = priceUpdater.latestPrices.USD; - blk.extras.medianTimestamp = block.medianTime; - blk.extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height); + const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); + + const blk: Partial = Object.assign({}, block); + const extras: Partial = {}; + + extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); + extras.coinbaseRaw = coinbaseTx.vin[0].scriptsig; + extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height); if (block.height === 0) { - blk.extras.medianFee = 0; // 50th percentiles - blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; - blk.extras.totalFees = 0; - blk.extras.avgFee = 0; - blk.extras.avgFeeRate = 0; - blk.extras.utxoSetChange = 0; - blk.extras.avgTxSize = 0; - blk.extras.totalInputs = 0; - blk.extras.totalOutputs = 1; - blk.extras.totalOutputAmt = 0; - blk.extras.segwitTotalTxs = 0; - blk.extras.segwitTotalSize = 0; - blk.extras.segwitTotalWeight = 0; + extras.medianFee = 0; // 50th percentiles + extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; + extras.totalFees = 0; + extras.avgFee = 0; + extras.avgFeeRate = 0; + extras.utxoSetChange = 0; + extras.avgTxSize = 0; + extras.totalInputs = 0; + extras.totalOutputs = 1; + extras.totalOutputAmt = 0; + extras.segwitTotalTxs = 0; + extras.segwitTotalSize = 0; + extras.segwitTotalWeight = 0; } else { - const stats = await bitcoinClient.getBlockStats(block.id); - blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles - blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); - blk.extras.totalFees = stats.totalfee; - blk.extras.avgFee = stats.avgfee; - blk.extras.avgFeeRate = stats.avgfeerate; - blk.extras.utxoSetChange = stats.utxo_increase; - blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01; - blk.extras.totalInputs = stats.ins; - blk.extras.totalOutputs = stats.outs; - blk.extras.totalOutputAmt = stats.total_out; - blk.extras.segwitTotalTxs = stats.swtxs; - blk.extras.segwitTotalSize = stats.swtotal_size; - blk.extras.segwitTotalWeight = stats.swtotal_weight; + const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id); + extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles + extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); + extras.totalFees = stats.totalfee; + extras.avgFee = stats.avgfee; + extras.avgFeeRate = stats.avgfeerate; + extras.utxoSetChange = stats.utxo_increase; + extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01; + extras.totalInputs = stats.ins; + extras.totalOutputs = stats.outs; + extras.totalOutputAmt = stats.total_out; + extras.segwitTotalTxs = stats.swtxs; + extras.segwitTotalSize = stats.swtotal_size; + extras.segwitTotalWeight = stats.swtotal_weight; } if (Common.blocksSummariesIndexingEnabled()) { - blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); - if (blk.extras.feePercentiles !== null) { - blk.extras.medianFeeAmt = blk.extras.feePercentiles[3]; + extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); + if (extras.feePercentiles !== null) { + extras.medianFeeAmt = extras.feePercentiles[3]; } } - blk.extras.virtualSize = block.weight / 4.0; - if (blk.extras.coinbaseTx.vout.length > 0) { - blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null; - blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null; - blk.extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(blk.extras.coinbaseTx.vin[0].scriptsig) ?? null; + extras.virtualSize = block.weight / 4.0; + if (coinbaseTx?.vout.length > 0) { + extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null; + extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null; + extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null; } else { - blk.extras.coinbaseAddress = null; - blk.extras.coinbaseSignature = null; - blk.extras.coinbaseSignatureAscii = null; + extras.coinbaseAddress = null; + extras.coinbaseSignature = null; + extras.coinbaseSignatureAscii = null; } const header = await bitcoinClient.getBlockHeader(block.id, false); - blk.extras.header = header; + extras.header = header; const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex'); if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) { const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height); - blk.extras.utxoSetSize = txoutset.txouts, - blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000); + extras.utxoSetSize = txoutset.txouts, + extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000); } else { - blk.extras.utxoSetSize = null; - blk.extras.totalInputAmt = null; + extras.utxoSetSize = null; + extras.totalInputAmt = null; } if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { let pool: PoolTag; - if (blk.extras?.coinbaseTx !== undefined) { - pool = await this.$findBlockMiner(blk.extras?.coinbaseTx); + if (coinbaseTx !== undefined) { + pool = await this.$findBlockMiner(coinbaseTx); } else { if (config.DATABASE.ENABLED === true) { pool = await poolsRepository.$getUnknownPool(); @@ -252,22 +252,24 @@ class Blocks { logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` + `Check your "pools" table entries`); } else { - blk.extras.pool = { - id: pool.id, + extras.pool = { + id: pool.uniqueId, name: pool.name, slug: pool.slug, }; } + extras.matchRate = null; if (config.MEMPOOL.AUDIT) { const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id); if (auditScore != null) { - blk.extras.matchRate = auditScore.matchRate; + extras.matchRate = auditScore.matchRate; } } } - return blk; + blk.extras = extras; + return blk; } /** @@ -293,15 +295,18 @@ class Blocks { } else { pools = poolsParser.miningPools; } + for (let i = 0; i < pools.length; ++i) { if (address !== undefined) { - const addresses: string[] = JSON.parse(pools[i].addresses); + const addresses: string[] = typeof pools[i].addresses === 'string' ? + JSON.parse(pools[i].addresses) : pools[i].addresses; if (addresses.indexOf(address) !== -1) { return pools[i]; } } - const regexes: string[] = JSON.parse(pools[i].regexes); + const regexes: string[] = typeof pools[i].regexes === 'string' ? + JSON.parse(pools[i].regexes) : pools[i].regexes; for (let y = 0; y < regexes.length; ++y) { const regex = new RegExp(regexes[y], 'i'); const match = asciiScriptSig.match(regex); @@ -652,7 +657,7 @@ class Blocks { if (Common.indexingEnabled()) { const dbBlock = await blocksRepository.$getBlockByHeight(height); if (dbBlock !== null) { - return prepareBlock(dbBlock); + return dbBlock; } } @@ -665,11 +670,11 @@ class Blocks { await blocksRepository.$saveBlockInDatabase(blockExtended); } - return prepareBlock(blockExtended); + return blockExtended; } /** - * Index a block by hash if it's missing from the database. Returns the block after indexing + * Get one block by its hash */ public async $getBlock(hash: string): Promise { // Check the memory cache @@ -678,29 +683,14 @@ class Blocks { return blockByHash; } - // Block has already been indexed - if (Common.indexingEnabled()) { - const dbBlock = await blocksRepository.$getBlockByHash(hash); - if (dbBlock !== null) { - return prepareBlock(dbBlock); - } - } - - // Not Bitcoin network, return the block as it + // Not Bitcoin network, return the block as it from the bitcoin backend if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { return await bitcoinApi.$getBlock(hash); } // Bitcoin network, add our custom data on top const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); - const transactions = await this.$getTransactionsExtended(hash, block.height, true); - const blockExtended = await this.$getBlockExtended(block, transactions); - if (Common.indexingEnabled()) { - delete(blockExtended['coinbaseTx']); - await blocksRepository.$saveBlockInDatabase(blockExtended); - } - - return blockExtended; + return await this.$indexBlock(block.height); } public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false, @@ -734,6 +724,18 @@ class Blocks { return summary.transactions; } + /** + * Get 15 blocks + * + * Internally this function uses two methods to get the blocks, and + * the method is automatically selected: + * - Using previous block hash links + * - Using block height + * + * @param fromHeight + * @param limit + * @returns + */ public async $getBlocks(fromHeight?: number, limit: number = 15): Promise { let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight; if (currentHeight > this.currentBlockHeight) { @@ -759,11 +761,14 @@ class Blocks { for (let i = 0; i < limit && currentHeight >= 0; i++) { let block = this.getBlocks().find((b) => b.height === currentHeight); if (block) { + // Using the memory cache (find by height) returnBlocks.push(block); } else if (Common.indexingEnabled()) { + // Using indexing (find by height, index on the fly, save in database) block = await this.$indexBlock(currentHeight); returnBlocks.push(block); - } else if (nextHash != null) { + } else if (nextHash !== null) { + // Without indexing, query block on the fly using bitoin backend, follow previous hash links block = await this.$indexBlock(currentHeight); nextHash = block.previousblockhash; returnBlocks.push(block); @@ -788,7 +793,7 @@ class Blocks { const blocks: any[] = []; while (fromHeight <= toHeight) { - let block: any = await blocksRepository.$getBlockByHeight(fromHeight); + let block: BlockExtended | null = await blocksRepository.$getBlockByHeight(fromHeight); if (!block) { await this.$indexBlock(fromHeight); block = await blocksRepository.$getBlockByHeight(fromHeight); @@ -801,11 +806,11 @@ class Blocks { const cleanBlock: any = { height: block.height ?? null, hash: block.id ?? null, - timestamp: block.blockTimestamp ?? null, - median_timestamp: block.medianTime ?? null, + timestamp: block.timestamp ?? null, + median_timestamp: block.mediantime ?? null, previous_block_hash: block.previousblockhash ?? null, difficulty: block.difficulty ?? null, - header: block.header ?? null, + header: block.extras.header ?? null, version: block.version ?? null, bits: block.bits ?? null, nonce: block.nonce ?? null, @@ -813,29 +818,30 @@ class Blocks { weight: block.weight ?? null, tx_count: block.tx_count ?? null, merkle_root: block.merkle_root ?? null, - reward: block.reward ?? null, - total_fee_amt: block.fees ?? null, - avg_fee_amt: block.avg_fee ?? null, - median_fee_amt: block.median_fee_amt ?? null, - fee_amt_percentiles: block.fee_percentiles ?? null, - avg_fee_rate: block.avg_fee_rate ?? null, - median_fee_rate: block.median_fee ?? null, - fee_rate_percentiles: block.fee_span ?? null, - total_inputs: block.total_inputs ?? null, - total_input_amt: block.total_input_amt ?? null, - total_outputs: block.total_outputs ?? null, - total_output_amt: block.total_output_amt ?? null, - segwit_total_txs: block.segwit_total_txs ?? null, - segwit_total_size: block.segwit_total_size ?? null, - segwit_total_weight: block.segwit_total_weight ?? null, - avg_tx_size: block.avg_tx_size ?? null, - utxoset_change: block.utxoset_change ?? null, - utxoset_size: block.utxoset_size ?? null, - coinbase_raw: block.coinbase_raw ?? null, - coinbase_address: block.coinbase_address ?? null, - coinbase_signature: block.coinbase_signature ?? null, - coinbase_signature_ascii: block.coinbase_signature_ascii ?? null, - pool_slug: block.pool_slug ?? null, + reward: block.extras.reward ?? null, + total_fee_amt: block.extras.totalFees ?? null, + avg_fee_amt: block.extras.avgFee ?? null, + median_fee_amt: block.extras.medianFeeAmt ?? null, + fee_amt_percentiles: block.extras.feePercentiles ?? null, + avg_fee_rate: block.extras.avgFeeRate ?? null, + median_fee_rate: block.extras.medianFee ?? null, + fee_rate_percentiles: block.extras.feeRange ?? null, + total_inputs: block.extras.totalInputs ?? null, + total_input_amt: block.extras.totalInputAmt ?? null, + total_outputs: block.extras.totalOutputs ?? null, + total_output_amt: block.extras.totalOutputAmt ?? null, + segwit_total_txs: block.extras.segwitTotalTxs ?? null, + segwit_total_size: block.extras.segwitTotalSize ?? null, + segwit_total_weight: block.extras.segwitTotalWeight ?? null, + avg_tx_size: block.extras.avgTxSize ?? null, + utxoset_change: block.extras.utxoSetChange ?? null, + utxoset_size: block.extras.utxoSetSize ?? null, + coinbase_raw: block.extras.coinbaseRaw ?? null, + coinbase_address: block.extras.coinbaseAddress ?? null, + coinbase_signature: block.extras.coinbaseSignature ?? null, + coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null, + pool_slug: block.extras.pool.slug ?? null, + pool_id: block.extras.pool.id ?? null, }; if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) { diff --git a/backend/src/api/chain-tips.ts b/backend/src/api/chain-tips.ts index 46a06f703..b68b0b281 100644 --- a/backend/src/api/chain-tips.ts +++ b/backend/src/api/chain-tips.ts @@ -43,7 +43,11 @@ class ChainTips { } } - public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] { + public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] { + if (height === undefined) { + return []; + } + const orphans: OrphanedBlock[] = []; for (const block of this.orphanedBlocks) { if (block.height === height) { diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index b34dcb7b8..0f009cd1d 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -7,6 +7,7 @@ import { PoolTag } from '../mempool.interfaces'; class PoolsParser { miningPools: any[] = []; unknownPool: any = { + 'id': 0, 'name': 'Unknown', 'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction', 'regexes': '[]', @@ -26,6 +27,7 @@ class PoolsParser { public setMiningPools(pools): void { for (const pool of pools) { pool.regexes = pool.tags; + pool.slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase(); delete(pool.tags); } this.miningPools = pools; diff --git a/backend/src/index.ts b/backend/src/index.ts index 6f259a2bd..7207ae7a8 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -179,7 +179,14 @@ class Server { setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); this.currentBackendRetryInterval = 5; } catch (e: any) { - const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`; + let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`; + loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`; + if (e?.stack) { + loggerMsg += ` Stack trace: ${e.stack}`; + } + // When we get a first Exception, only `logger.debug` it and retry after 5 seconds + // From the second Exception, `logger.warn` the Exception and increase the retry delay + // Maximum retry delay is 60 seconds if (this.currentBackendRetryInterval > 5) { logger.warn(loggerMsg); mempool.setOutOfSync(); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index cb95be98a..a7937e01d 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -1,9 +1,10 @@ import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; import { OrphanedBlock } from './api/chain-tips'; -import { HeapNode } from "./utils/pairing-heap"; +import { HeapNode } from './utils/pairing-heap'; export interface PoolTag { - id: number; // mysql row id + id: number; + uniqueId: number; name: string; link: string; regexes: string; // JSON array @@ -147,44 +148,44 @@ export interface TransactionStripped { } export interface BlockExtension { - totalFees?: number; - medianFee?: number; - feeRange?: number[]; - reward?: number; - coinbaseTx?: TransactionMinerInfo; - matchRate?: number; - pool?: { - id: number; + totalFees: number; + medianFee: number; // median fee rate + feeRange: number[]; // fee rate percentiles + reward: number; + matchRate: number | null; + pool: { + id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id` name: string; slug: string; }; - avgFee?: number; - avgFeeRate?: number; - coinbaseRaw?: string; - usd?: number | null; - medianTimestamp?: number; - blockTime?: number; - orphans?: OrphanedBlock[] | null; - coinbaseAddress?: string | null; - coinbaseSignature?: string | null; - coinbaseSignatureAscii?: string | null; - virtualSize?: number; - avgTxSize?: number; - totalInputs?: number; - totalOutputs?: number; - totalOutputAmt?: number; - medianFeeAmt?: number | null; - feePercentiles?: number[] | null, - segwitTotalTxs?: number; - segwitTotalSize?: number; - segwitTotalWeight?: number; - header?: string; - utxoSetChange?: number; + avgFee: number; + avgFeeRate: number; + coinbaseRaw: string; + orphans: OrphanedBlock[] | null; + coinbaseAddress: string | null; + coinbaseSignature: string | null; + coinbaseSignatureAscii: string | null; + virtualSize: number; + avgTxSize: number; + totalInputs: number; + totalOutputs: number; + totalOutputAmt: number; + medianFeeAmt: number | null; // median fee in sats + feePercentiles: number[] | null, // fee percentiles in sats + segwitTotalTxs: number; + segwitTotalSize: number; + segwitTotalWeight: number; + header: string; + utxoSetChange: number; // Requires coinstatsindex, will be set to NULL otherwise - utxoSetSize?: number | null; - totalInputAmt?: number | null; + utxoSetSize: number | null; + totalInputAmt: number | null; } +/** + * Note: Everything that is added in here will be automatically returned through + * /api/v1/block and /api/v1/blocks APIs + */ export interface BlockExtended extends IEsploraApi.Block { extras: BlockExtension; } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index c7edb97cb..ac12b3430 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,8 +1,7 @@ -import { BlockExtended, BlockPrice } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces'; import DB from '../database'; import logger from '../logger'; import { Common } from '../api/common'; -import { prepareBlock } from '../utils/blocks-utils'; import PoolsRepository from './PoolsRepository'; import HashratesRepository from './HashratesRepository'; import { escape } from 'mysql2'; @@ -10,6 +9,50 @@ import BlocksSummariesRepository from './BlocksSummariesRepository'; import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository'; import bitcoinClient from '../api/bitcoin/bitcoin-client'; import config from '../config'; +import chainTips from '../api/chain-tips'; +import blocks from '../api/blocks'; + +const BLOCK_DB_FIELDS = ` + blocks.hash AS id, + blocks.height, + blocks.version, + UNIX_TIMESTAMP(blocks.blockTimestamp) AS timestamp, + blocks.bits, + blocks.nonce, + blocks.difficulty, + blocks.merkle_root, + blocks.tx_count, + blocks.size, + blocks.weight, + blocks.previous_block_hash AS previousblockhash, + UNIX_TIMESTAMP(blocks.median_timestamp) AS mediantime, + blocks.fees AS totalFees, + blocks.median_fee AS medianFee, + blocks.fee_span AS feeRange, + blocks.reward, + pools.unique_id AS poolId, + pools.name AS poolName, + pools.slug AS poolSlug, + blocks.avg_fee AS avgFee, + blocks.avg_fee_rate AS avgFeeRate, + blocks.coinbase_raw AS coinbaseRaw, + blocks.coinbase_address AS coinbaseAddress, + blocks.coinbase_signature AS coinbaseSignature, + blocks.coinbase_signature_ascii AS coinbaseSignatureAscii, + blocks.avg_tx_size AS avgTxSize, + blocks.total_inputs AS totalInputs, + blocks.total_outputs AS totalOutputs, + blocks.total_output_amt AS totalOutputAmt, + blocks.median_fee_amt AS medianFeeAmt, + blocks.fee_percentiles AS feePercentiles, + blocks.segwit_total_txs AS segwitTotalTxs, + blocks.segwit_total_size AS segwitTotalSize, + blocks.segwit_total_weight AS segwitTotalWeight, + blocks.header, + blocks.utxoset_change AS utxoSetChange, + blocks.utxoset_size AS utxoSetSize, + blocks.total_input_amt AS totalInputAmts +`; class BlocksRepository { /** @@ -44,6 +87,11 @@ class BlocksRepository { ?, ? )`; + const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id); + if (!poolDbId) { + throw Error(`Could not find a mining pool with the unique_id = ${block.extras.pool.id}. This error should never be printed.`); + } + const params: any[] = [ block.height, block.id, @@ -53,7 +101,7 @@ class BlocksRepository { block.tx_count, block.extras.coinbaseRaw, block.difficulty, - block.extras.pool?.id, // Should always be set to something + poolDbId.id, block.extras.totalFees, JSON.stringify(block.extras.feeRange), block.extras.medianFee, @@ -65,7 +113,7 @@ class BlocksRepository { block.previousblockhash, block.extras.avgFee, block.extras.avgFeeRate, - block.extras.medianTimestamp, + block.mediantime, block.extras.header, block.extras.coinbaseAddress, truncatedCoinbaseSignature, @@ -87,9 +135,9 @@ class BlocksRepository { await DB.query(query, params); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart - logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`); + logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`, logger.tags.mining); } else { - logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); throw e; } } @@ -307,34 +355,17 @@ class BlocksRepository { /** * Get blocks mined by a specific mining pool */ - public async $getBlocksByPool(slug: string, startHeight?: number): Promise { + public async $getBlocksByPool(slug: string, startHeight?: number): Promise { const pool = await PoolsRepository.$getPool(slug); if (!pool) { throw new Error('This mining pool does not exist ' + escape(slug)); } const params: any[] = []; - let query = ` SELECT - blocks.height, - hash as id, - UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, - size, - weight, - tx_count, - coinbase_raw, - difficulty, - fees, - fee_span, - median_fee, - reward, - version, - bits, - nonce, - merkle_root, - previous_block_hash as previousblockhash, - avg_fee, - avg_fee_rate + let query = ` + SELECT ${BLOCK_DB_FIELDS} FROM blocks + JOIN pools ON blocks.pool_id = pools.id WHERE pool_id = ?`; params.push(pool.id); @@ -347,11 +378,11 @@ class BlocksRepository { LIMIT 10`; try { - const [rows] = await DB.query(query, params); + const [rows]: any[] = await DB.query(query, params); const blocks: BlockExtended[] = []; - for (const block of rows) { - blocks.push(prepareBlock(block)); + for (const block of rows) { + blocks.push(await this.formatDbBlockIntoExtendedBlock(block)); } return blocks; @@ -364,32 +395,21 @@ class BlocksRepository { /** * Get one block by height */ - public async $getBlockByHeight(height: number): Promise { + public async $getBlockByHeight(height: number): Promise { try { - const [rows]: any[] = await DB.query(`SELECT - blocks.*, - hash as id, - UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, - UNIX_TIMESTAMP(blocks.median_timestamp) as medianTime, - pools.id as pool_id, - pools.name as pool_name, - pools.link as pool_link, - pools.slug as pool_slug, - pools.addresses as pool_addresses, - pools.regexes as pool_regexes, - previous_block_hash as previousblockhash + const [rows]: any[] = await DB.query(` + SELECT ${BLOCK_DB_FIELDS} FROM blocks JOIN pools ON blocks.pool_id = pools.id - WHERE blocks.height = ${height} - `); + WHERE blocks.height = ?`, + [height] + ); if (rows.length <= 0) { return null; } - rows[0].fee_span = JSON.parse(rows[0].fee_span); - rows[0].fee_percentiles = JSON.parse(rows[0].fee_percentiles); - return rows[0]; + return await this.formatDbBlockIntoExtendedBlock(rows[0]); } catch (e) { logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e)); throw e; @@ -402,10 +422,7 @@ class BlocksRepository { public async $getBlockByHash(hash: string): Promise { try { const query = ` - SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id, - pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug, - pools.addresses as pool_addresses, pools.regexes as pool_regexes, - previous_block_hash as previousblockhash + SELECT ${BLOCK_DB_FIELDS} FROM blocks JOIN pools ON blocks.pool_id = pools.id WHERE hash = ?; @@ -415,9 +432,8 @@ class BlocksRepository { if (rows.length <= 0) { return null; } - - rows[0].fee_span = JSON.parse(rows[0].fee_span); - return rows[0]; + + return await this.formatDbBlockIntoExtendedBlock(rows[0]); } catch (e) { logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e)); throw e; @@ -833,6 +849,86 @@ class BlocksRepository { throw e; } } + + /** + * Convert a mysql row block into a BlockExtended. Note that you + * must provide the correct field into dbBlk object param + * + * @param dbBlk + */ + private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise { + const blk: Partial = {}; + const extras: Partial = {}; + + // IEsploraApi.Block + blk.id = dbBlk.id; + blk.height = dbBlk.height; + blk.version = dbBlk.version; + blk.timestamp = dbBlk.timestamp; + blk.bits = dbBlk.bits; + blk.nonce = dbBlk.nonce; + blk.difficulty = dbBlk.difficulty; + blk.merkle_root = dbBlk.merkle_root; + blk.tx_count = dbBlk.tx_count; + blk.size = dbBlk.size; + blk.weight = dbBlk.weight; + blk.previousblockhash = dbBlk.previousblockhash; + blk.mediantime = dbBlk.mediantime; + + // BlockExtension + extras.totalFees = dbBlk.totalFees; + extras.medianFee = dbBlk.medianFee; + extras.feeRange = JSON.parse(dbBlk.feeRange); + extras.reward = dbBlk.reward; + extras.pool = { + id: dbBlk.poolId, + name: dbBlk.poolName, + slug: dbBlk.poolSlug, + }; + extras.avgFee = dbBlk.avgFee; + extras.avgFeeRate = dbBlk.avgFeeRate; + extras.coinbaseRaw = dbBlk.coinbaseRaw; + extras.coinbaseAddress = dbBlk.coinbaseAddress; + extras.coinbaseSignature = dbBlk.coinbaseSignature; + extras.coinbaseSignatureAscii = dbBlk.coinbaseSignatureAscii; + extras.avgTxSize = dbBlk.avgTxSize; + extras.totalInputs = dbBlk.totalInputs; + extras.totalOutputs = dbBlk.totalOutputs; + extras.totalOutputAmt = dbBlk.totalOutputAmt; + extras.medianFeeAmt = dbBlk.medianFeeAmt; + extras.feePercentiles = JSON.parse(dbBlk.feePercentiles); + extras.segwitTotalTxs = dbBlk.segwitTotalTxs; + extras.segwitTotalSize = dbBlk.segwitTotalSize; + extras.segwitTotalWeight = dbBlk.segwitTotalWeight; + extras.header = dbBlk.header, + extras.utxoSetChange = dbBlk.utxoSetChange; + extras.utxoSetSize = dbBlk.utxoSetSize; + extras.totalInputAmt = dbBlk.totalInputAmt; + extras.virtualSize = dbBlk.weight / 4.0; + + // Re-org can happen after indexing so we need to always get the + // latest state from core + extras.orphans = chainTips.getOrphanedBlocksAtHeight(dbBlk.height); + + // If we're missing block summary related field, check if we can populate them on the fly now + if (Common.blocksSummariesIndexingEnabled() && + (extras.medianFeeAmt === null || extras.feePercentiles === null)) + { + extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); + if (extras.feePercentiles === null) { + const block = await bitcoinClient.getBlock(dbBlk.id, 2); + const summary = blocks.summarizeBlock(block); + await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary }); + extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); + } + if (extras.feePercentiles !== null) { + extras.medianFeeAmt = extras.feePercentiles[3]; + } + } + + blk.extras = extras; + return blk; + } } export default new BlocksRepository(); diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index 236955d65..293fd5e39 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -10,7 +10,7 @@ class PoolsRepository { * Get all pools tagging info */ public async $getPools(): Promise { - const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;'); + const [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, addresses, regexes, slug FROM pools'); return rows; } @@ -18,10 +18,10 @@ class PoolsRepository { * Get unknown pool tagging info */ public async $getUnknownPool(): Promise { - let [rows]: any[] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"'); + let [rows]: any[] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"'); if (rows && rows.length === 0 && config.DATABASE.ENABLED) { await poolsParser.$insertUnknownPool(); - [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"'); + [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"'); } return rows[0]; } diff --git a/backend/src/utils/blocks-utils.ts b/backend/src/utils/blocks-utils.ts deleted file mode 100644 index 43a2fc964..000000000 --- a/backend/src/utils/blocks-utils.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { BlockExtended } from '../mempool.interfaces'; - -export function prepareBlock(block: any): BlockExtended { - return { - id: block.id ?? block.hash, // hash for indexed block - timestamp: block.timestamp ?? block.time ?? block.blockTimestamp, // blockTimestamp for indexed block - height: block.height, - version: block.version, - bits: (typeof block.bits === 'string' ? parseInt(block.bits, 16): block.bits), - nonce: block.nonce, - difficulty: block.difficulty, - merkle_root: block.merkle_root ?? block.merkleroot, - tx_count: block.tx_count ?? block.nTx, - size: block.size, - weight: block.weight, - previousblockhash: block.previousblockhash, - extras: { - coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw, - medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee, - feeRange: block.feeRange ?? block?.extras?.feeRange ?? block.fee_span, - reward: block.reward ?? block?.extras?.reward, - totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees, - avgFee: block?.extras?.avgFee ?? block.avg_fee, - avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate, - pool: block?.extras?.pool ?? (block?.pool_id ? { - id: block.pool_id, - name: block.pool_name, - slug: block.pool_slug, - } : undefined), - usd: block?.extras?.usd ?? block.usd ?? null, - } - }; -} diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 8fa30a723..7ed32a9de 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -114,7 +114,6 @@ export interface BlockExtension { medianFee?: number; feeRange?: number[]; reward?: number; - coinbaseTx?: Transaction; coinbaseRaw?: string; matchRate?: number; pool?: { From 01d699e454648a97a4f374e120f3276666691461 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 27 Feb 2023 18:39:02 +0900 Subject: [PATCH 3/5] Add missing match rate to the block returned from the database --- backend/src/repositories/BlocksRepository.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index ac12b3430..9cd31bbab 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -11,6 +11,7 @@ import bitcoinClient from '../api/bitcoin/bitcoin-client'; import config from '../config'; import chainTips from '../api/chain-tips'; import blocks from '../api/blocks'; +import BlocksAuditsRepository from './BlocksAuditsRepository'; const BLOCK_DB_FIELDS = ` blocks.hash AS id, @@ -910,6 +911,15 @@ class BlocksRepository { // latest state from core extras.orphans = chainTips.getOrphanedBlocksAtHeight(dbBlk.height); + // Match rate is not part of the blocks table, but it is part of APIs so we must include it + extras.matchRate = null; + if (config.MEMPOOL.AUDIT) { + const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(dbBlk.id); + if (auditScore != null) { + extras.matchRate = auditScore.matchRate; + } + } + // If we're missing block summary related field, check if we can populate them on the fly now if (Common.blocksSummariesIndexingEnabled() && (extras.medianFeeAmt === null || extras.feePercentiles === null)) From 76ae9d4ccba76ba8d8a854d9ce39474185bf273a Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 27 Feb 2023 19:06:46 +0900 Subject: [PATCH 4/5] Wipe the disk cache since we have a new block structure --- backend/src/api/disk-cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/disk-cache.ts b/backend/src/api/disk-cache.ts index a75fd43cc..2a53e7b9b 100644 --- a/backend/src/api/disk-cache.ts +++ b/backend/src/api/disk-cache.ts @@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces'; import { Common } from './common'; class DiskCache { - private cacheSchemaVersion = 2; + private cacheSchemaVersion = 3; private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json'; private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json'; From a67656389ea75f591b90f9164b717d7629038a66 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 1 Mar 2023 13:50:15 +0900 Subject: [PATCH 5/5] Fix chain divergence detection upon new block (use the new interface) --- backend/src/api/blocks.ts | 14 +++++++------- backend/src/repositories/BlocksRepository.ts | 11 +++++++++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 75d1ec300..12eb3b693 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -570,18 +570,18 @@ class Blocks { if (Common.indexingEnabled()) { if (!fastForwarded) { const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1); - if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) { - logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`); + if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) { + logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`); // We assume there won't be a reorg with more than 10 block depth - await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10); + await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10); await HashratesRepository.$deleteLastEntries(); - await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10); - await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10); + await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10); + await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10); for (let i = 10; i >= 0; --i) { - const newBlock = await this.$indexBlock(lastBlock['height'] - i); + const newBlock = await this.$indexBlock(lastBlock.height - i); await this.$getStrippedBlockTransactions(newBlock.id, true, true); if (config.MEMPOOL.CPFP_INDEXING) { - await this.$indexCPFP(newBlock.id, lastBlock['height'] - i); + await this.$indexCPFP(newBlock.id, lastBlock.height - i); } } await mining.$indexDifficultyAdjustments(); diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 9cd31bbab..80df1ac92 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -525,8 +525,15 @@ class BlocksRepository { public async $validateChain(): Promise { try { const start = new Date().getTime(); - const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash, - UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`); + const [blocks]: any[] = await DB.query(` + SELECT + height, + hash, + previous_block_hash, + UNIX_TIMESTAMP(blockTimestamp) AS timestamp + FROM blocks + ORDER BY height + `); let partialMsg = false; let idx = 1;