mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2024-12-29 09:44:26 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1330 from nymkappa/feature/index-more-data

Browse Source 
Index more data using getblockstats core RPC
This commit is contained in:
softsimon 2022-03-11 14:05:03 +01:00 committed by GitHub
commit 5caaa1633a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 56 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
backend/src/api/blocks.ts
View File

@ -108,17 +108,14 @@ class Blocks {
blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
const transactionsTmp = [...transactions];
transactionsTmp.shift();
transactionsTmp.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
blockExtended.extras.medianFee = transactionsTmp.length > 0 ?
Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0;
blockExtended.extras.feeRange = transactionsTmp.length > 0 ?
Common.getFeesInRange(transactionsTmp, 8) : [0, 0];
blockExtended.extras.totalFees = transactionsTmp.reduce((acc, tx) => {
return acc + tx.fee;
}, 0)
const stats = await bitcoinClient.getBlockStats(block.id);
const coinbaseRaw: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
blockExtended.extras.coinbaseRaw = coinbaseRaw.hex;
blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
blockExtended.extras.totalFees = stats.totalfee;
blockExtended.extras.avgFee = stats.avgfee;
blockExtended.extras.avgFeeRate = stats.avgfeerate;
if (Common.indexingEnabled()) {
let pool: PoolTag;
@ -184,7 +181,6 @@ class Blocks {
}
this.blockIndexingStarted = true;
const startedAt = new Date().getTime() / 1000;
try {
let currentBlockHeight = blockchainInfo.blocks;
@ -201,6 +197,9 @@ class Blocks {
const chunkSize = 10000;
let totaIndexed = await blocksRepository.$blockCount(null, null);
let indexedThisRun = 0;
const startedAt = new Date().getTime() / 1000;
let timer = new Date().getTime() / 1000;
while (currentBlockHeight >= lastBlockToIndex) {
const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
@ -219,12 +218,16 @@ class Blocks {
break;
}
++indexedThisRun;
if (++totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) {
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
++totaIndexed;
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
const progress = Math.round(totaIndexed / indexingBlockAmount * 100);
const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds);
logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${elapsedSeconds} seconds | left: ~${timeLeft} seconds`);
logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
timer = new Date().getTime() / 1000;
indexedThisRun = 0;
}
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block = await bitcoinApi.$getBlock(blockHash);

22
backend/src/api/database-migration.ts
View File

@ -6,7 +6,7 @@ import logger from '../logger';
const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
class DatabaseMigration {
private static currentVersion = 10;
private static currentVersion = 11;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
@ -92,13 +92,13 @@ class DatabaseMigration {
await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
}
if (databaseSchemaVersion < 5 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.'`);
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 6 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.'`);
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
// Cleanup original blocks fields type
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
@ -125,7 +125,7 @@ class DatabaseMigration {
}
if (databaseSchemaVersion < 8 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.'`);
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
@ -134,7 +134,7 @@ class DatabaseMigration {
}
if (databaseSchemaVersion < 9 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.'`);
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
@ -144,6 +144,18 @@ class DatabaseMigration {
await this.$executeQuery(connection, 'ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
}
if (databaseSchemaVersion < 11 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery(connection, `ALTER TABLE blocks
ADD avg_fee INT UNSIGNED NULL,
ADD avg_fee_rate INT UNSIGNED NULL
`);
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
}
connection.release();
} catch (e) {
connection.release();

7
backend/src/mempool.interfaces.ts
View File

@ -79,7 +79,7 @@ export interface TransactionStripped {
export interface BlockExtension {
totalFees?: number;
medianFee?: number;
medianFee?: number; // Actually the median fee rate that we compute ourself
feeRange?: number[];
reward?: number;
coinbaseTx?: TransactionMinerInfo;
@ -87,7 +87,10 @@ export interface BlockExtension {
pool?: {
id: number;
name: string;
}
};
avgFee?: number;
avgFeeRate?: number;
coinbaseRaw?: string;
}
export interface BlockExtended extends IEsploraApi.Block {

31
backend/src/repositories/BlocksRepository.ts
View File

@ -12,17 +12,17 @@ class BlocksRepository {
try {
const query = `INSERT INTO blocks(
height, hash, blockTimestamp, size,
weight, tx_count, coinbase_raw, difficulty,
pool_id, fees, fee_span, median_fee,
reward, version, bits, nonce,
merkle_root, previous_block_hash
height, hash, blockTimestamp, size,
weight, tx_count, coinbase_raw, difficulty,
pool_id, fees, fee_span, median_fee,
reward, version, bits, nonce,
merkle_root, previous_block_hash, avg_fee, avg_fee_rate
) VALUE (
?, ?, FROM_UNIXTIME(?), ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?
?, ?, ?, ?
)`;
const params: any[] = [
@ -32,21 +32,22 @@ class BlocksRepository {
block.size,
block.weight,
block.tx_count,
'',
block.extras.coinbaseRaw,
block.difficulty,
block.extras.pool?.id, // Should always be set to something
0,
'[]',
block.extras.medianFee ?? 0,
block.extras.reward ?? 0,
block.extras.totalFees,
JSON.stringify(block.extras.feeRange),
block.extras.medianFee,
block.extras.reward,
block.version,
block.bits,
block.nonce,
block.merkle_root,
block.previousblockhash
block.previousblockhash,
block.extras.avgFee,
block.extras.avgFeeRate,
];
// logger.debug(query);
await connection.query(query, params);
connection.release();
} catch (e: any) {
@ -272,7 +273,7 @@ class BlocksRepository {
/**
* Get one block by height
*/
public async $getBlockByHeight(height: number): Promise<object | null> {
public async $getBlockByHeight(height: number): Promise<object | null> {
const connection = await DB.pool.getConnection();
try {
const [rows]: any[] = await connection.query(`
@ -298,7 +299,7 @@ class BlocksRepository {
/**
* Return blocks difficulty
*/
public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
interval = Common.getSqlInterval(interval);
const connection = await DB.pool.getConnection();