From d82f9c499893adefc90db1db497193627b0fe595 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 9 Mar 2022 19:18:51 +0100 Subject: [PATCH 1/3] Index more data using getblockstats core RPC --- backend/src/api/blocks.ts | 18 ++++++--- backend/src/api/database-migration.ts | 25 ++++++++++--- backend/src/mempool.interfaces.ts | 12 +++++- backend/src/repositories/BlocksRepository.ts | 39 +++++++++++++------- 4 files changed, 67 insertions(+), 27 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index f0c04455b..3e84e1390 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -111,14 +111,20 @@ class Blocks { const transactionsTmp = [...transactions]; transactionsTmp.shift(); transactionsTmp.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize); - blockExtended.extras.medianFee = transactionsTmp.length > 0 ? Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0; - blockExtended.extras.feeRange = transactionsTmp.length > 0 ? - Common.getFeesInRange(transactionsTmp, 8) : [0, 0]; - blockExtended.extras.totalFees = transactionsTmp.reduce((acc, tx) => { - return acc + tx.fee; - }, 0) + + const stats = await bitcoinClient.getBlockStats(block.id); + blockExtended.extras.feeRange = stats.feerate_percentiles; + blockExtended.extras.totalFees = stats.totalfee; + blockExtended.extras.avgFee = stats.avgfee; + blockExtended.extras.avgFeeRate = stats.avgfeerate; + blockExtended.extras.maxFee = stats.maxfee; + blockExtended.extras.maxFeeRate = stats.maxfeerate; + blockExtended.extras.minFee = stats.minfee; + blockExtended.extras.minFeeRate = stats.minfeerate; + blockExtended.extras.subsidy = stats.subsidy; + blockExtended.extras.medianFeeValue = stats.medianfee; if (Common.indexingEnabled()) { let pool: PoolTag; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 9af507461..0331d2f96 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,7 +6,7 @@ import logger from '../logger'; const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 10; + private static currentVersion = 11; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -92,13 +92,13 @@ class DatabaseMigration { await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks')); } if (databaseSchemaVersion < 5 && isBitcoin === true) { - logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.'`); + logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`); await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"'); } if (databaseSchemaVersion < 6 && isBitcoin === true) { - logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.'`); + logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`); await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index // Cleanup original blocks fields type await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"'); @@ -125,7 +125,7 @@ class DatabaseMigration { } if (databaseSchemaVersion < 8 && isBitcoin === true) { - logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.'`); + logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`); await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index await this.$executeQuery(connection, 'ALTER TABLE `hashrates` DROP INDEX `PRIMARY`'); await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST'); @@ -134,7 +134,7 @@ class DatabaseMigration { } if (databaseSchemaVersion < 9 && isBitcoin === true) { - logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.'`); + logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`); await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)'); await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)'); @@ -144,6 +144,21 @@ class DatabaseMigration { await this.$executeQuery(connection, 'ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)'); } + if (databaseSchemaVersion < 11 && isBitcoin === true) { + logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`); + await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index + await this.$executeQuery(connection, `ALTER TABLE blocks + ADD avg_fee int unsigned NULL, + ADD avg_fee_rate int unsigned NULL, + ADD max_fee int unsigned NULL, + ADD max_fee_rate int unsigned NULL, + ADD min_fee int unsigned NULL, + ADD min_fee_rate int unsigned NULL, + ADD median_fee_value int unsigned NULL, + ADD subsidy float unsigned NULL; + `); + } + connection.release(); } catch (e) { connection.release(); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 810398cab..e99b1efb6 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -79,7 +79,7 @@ export interface TransactionStripped { export interface BlockExtension { totalFees?: number; - medianFee?: number; + medianFee?: number; // Actually the median fee rate that we compute ourself feeRange?: number[]; reward?: number; coinbaseTx?: TransactionMinerInfo; @@ -87,7 +87,15 @@ export interface BlockExtension { pool?: { id: number; name: string; - } + }; + avgFee?: number; + avgFeeRate?: number; + maxFee?: number; + maxFeeRate?: number; + minFee?: number; + minFeeRate?: number; + subsidy?: number; + medianFeeValue?: number; // The actual median fee amount from getblockstats RPC } export interface BlockExtended extends IEsploraApi.Block { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 2364a8485..46ca4935e 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -12,17 +12,21 @@ class BlocksRepository { try { const query = `INSERT INTO blocks( - height, hash, blockTimestamp, size, - weight, tx_count, coinbase_raw, difficulty, - pool_id, fees, fee_span, median_fee, - reward, version, bits, nonce, - merkle_root, previous_block_hash + height, hash, blockTimestamp, size, + weight, tx_count, coinbase_raw, difficulty, + pool_id, fees, fee_span, median_fee, + reward, version, bits, nonce, + merkle_root, previous_block_hash, avg_fee, avg_fee_rate, + max_fee, max_fee_rate, min_fee, min_fee_rate, + median_fee_value, subsidy ) VALUE ( ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ? + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ? )`; const params: any[] = [ @@ -35,18 +39,25 @@ class BlocksRepository { '', block.difficulty, block.extras.pool?.id, // Should always be set to something - 0, - '[]', - block.extras.medianFee ?? 0, - block.extras.reward ?? 0, + block.extras.totalFees, + JSON.stringify(block.extras.feeRange), + block.extras.medianFee, + block.extras.reward, block.version, block.bits, block.nonce, block.merkle_root, - block.previousblockhash + block.previousblockhash, + block.extras.avgFee, + block.extras.avgFeeRate, + block.extras.maxFee, + block.extras.maxFeeRate, + block.extras.minFee, + block.extras.minFeeRate, + block.extras.medianFeeValue, + block.extras.subsidy, ]; - // logger.debug(query); await connection.query(query, params); connection.release(); } catch (e: any) { @@ -272,7 +283,7 @@ class BlocksRepository { /** * Get one block by height */ - public async $getBlockByHeight(height: number): Promise { + public async $getBlockByHeight(height: number): Promise { const connection = await DB.pool.getConnection(); try { const [rows]: any[] = await connection.query(` @@ -298,7 +309,7 @@ class BlocksRepository { /** * Return blocks difficulty */ - public async $getBlocksDifficulty(interval: string | null): Promise { + public async $getBlocksDifficulty(interval: string | null): Promise { interval = Common.getSqlInterval(interval); const connection = await DB.pool.getConnection(); From 8ca3f6e72b93770eb2a5dc829e492647dbbe95be Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 10 Mar 2022 14:21:11 +0100 Subject: [PATCH 2/3] Index more data using getblockstats core RPC --- backend/src/api/blocks.ts | 32 +++++++++----------- backend/src/api/database-migration.ts | 13 +++----- backend/src/mempool.interfaces.ts | 7 +---- backend/src/repositories/BlocksRepository.ts | 16 ++-------- 4 files changed, 23 insertions(+), 45 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 3e84e1390..f4319c04c 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -108,23 +108,14 @@ class Blocks { blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); - const transactionsTmp = [...transactions]; - transactionsTmp.shift(); - transactionsTmp.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize); - blockExtended.extras.medianFee = transactionsTmp.length > 0 ? - Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0; - const stats = await bitcoinClient.getBlockStats(block.id); - blockExtended.extras.feeRange = stats.feerate_percentiles; + const coinbaseRaw: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); + blockExtended.extras.coinbaseRaw = coinbaseRaw.hex; + blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles + blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); blockExtended.extras.totalFees = stats.totalfee; blockExtended.extras.avgFee = stats.avgfee; blockExtended.extras.avgFeeRate = stats.avgfeerate; - blockExtended.extras.maxFee = stats.maxfee; - blockExtended.extras.maxFeeRate = stats.maxfeerate; - blockExtended.extras.minFee = stats.minfee; - blockExtended.extras.minFeeRate = stats.minfeerate; - blockExtended.extras.subsidy = stats.subsidy; - blockExtended.extras.medianFeeValue = stats.medianfee; if (Common.indexingEnabled()) { let pool: PoolTag; @@ -190,7 +181,6 @@ class Blocks { } this.blockIndexingStarted = true; - const startedAt = new Date().getTime() / 1000; try { let currentBlockHeight = blockchainInfo.blocks; @@ -207,6 +197,9 @@ class Blocks { const chunkSize = 10000; let totaIndexed = await blocksRepository.$blockCount(null, null); let indexedThisRun = 0; + const startedAt = new Date().getTime() / 1000; + let timer = new Date().getTime() / 1000; + while (currentBlockHeight >= lastBlockToIndex) { const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1); @@ -225,13 +218,16 @@ class Blocks { break; } ++indexedThisRun; - if (++totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) { - const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); + const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); + if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) { + const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); const progress = Math.round(totaIndexed / indexingBlockAmount * 100); const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds); - logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${elapsedSeconds} seconds | left: ~${timeLeft} seconds`); - } + logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + timer = new Date().getTime() / 1000; + indexedThisRun = 0; + } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); const block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true); diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 0331d2f96..39ed12bb0 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -148,15 +148,12 @@ class DatabaseMigration { logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`); await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index await this.$executeQuery(connection, `ALTER TABLE blocks - ADD avg_fee int unsigned NULL, - ADD avg_fee_rate int unsigned NULL, - ADD max_fee int unsigned NULL, - ADD max_fee_rate int unsigned NULL, - ADD min_fee int unsigned NULL, - ADD min_fee_rate int unsigned NULL, - ADD median_fee_value int unsigned NULL, - ADD subsidy float unsigned NULL; + ADD avg_fee INT UNSIGNED NULL, + ADD avg_fee_rate INT UNSIGNED NULL `); + await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); + await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"'); + await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"'); } connection.release(); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index e99b1efb6..bda702ccb 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -90,12 +90,7 @@ export interface BlockExtension { }; avgFee?: number; avgFeeRate?: number; - maxFee?: number; - maxFeeRate?: number; - minFee?: number; - minFeeRate?: number; - subsidy?: number; - medianFeeValue?: number; // The actual median fee amount from getblockstats RPC + coinbaseRaw?: string; } export interface BlockExtended extends IEsploraApi.Block { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 46ca4935e..0cab3c0db 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -16,17 +16,13 @@ class BlocksRepository { weight, tx_count, coinbase_raw, difficulty, pool_id, fees, fee_span, median_fee, reward, version, bits, nonce, - merkle_root, previous_block_hash, avg_fee, avg_fee_rate, - max_fee, max_fee_rate, min_fee, min_fee_rate, - median_fee_value, subsidy + merkle_root, previous_block_hash, avg_fee, avg_fee_rate ) VALUE ( ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, - ?, ?, ?, ?, - ?, ? + ?, ?, ?, ? )`; const params: any[] = [ @@ -36,7 +32,7 @@ class BlocksRepository { block.size, block.weight, block.tx_count, - '', + block.extras.coinbaseRaw, block.difficulty, block.extras.pool?.id, // Should always be set to something block.extras.totalFees, @@ -50,12 +46,6 @@ class BlocksRepository { block.previousblockhash, block.extras.avgFee, block.extras.avgFeeRate, - block.extras.maxFee, - block.extras.maxFeeRate, - block.extras.minFee, - block.extras.minFeeRate, - block.extras.medianFeeValue, - block.extras.subsidy, ]; await connection.query(query, params); From 9a71c15b4949c48a10ff442bf515ce81c2cfbabc Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 10 Mar 2022 14:23:29 +0100 Subject: [PATCH 3/3] Fix block indexing log --- backend/src/api/blocks.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index f4319c04c..79b11bcb1 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -218,6 +218,7 @@ class Blocks { break; } ++indexedThisRun; + ++totaIndexed; const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); @@ -227,7 +228,7 @@ class Blocks { logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); timer = new Date().getTime() / 1000; indexedThisRun = 0; - } + } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); const block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);