mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2025-02-25 07:07:36 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3166 from mempool/nymkappa/unify-blocks-apis

Browse source
This commit is contained in:
wiz 2023-03-01 19:56:02 +09:00 • committed by GitHub
commit 4efabe18b1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 394 additions and 264 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
backend/src/api/bitcoin/bitcoin-api.interface.ts
View file

@ -172,4 +172,35 @@ export namespace IBitcoinApi {
} }
} }
export interface BlockStats {
"avgfee": number;
"avgfeerate": number;
"avgtxsize": number;
"blockhash": string;
"feerate_percentiles": [number, number, number, number, number];
"height": number;
"ins": number;
"maxfee": number;
"maxfeerate": number;
"maxtxsize": number;
"medianfee": number;
"mediantime": number;
"mediantxsize": number;
"minfee": number;
"minfeerate": number;
"mintxsize": number;
"outs": number;
"subsidy": number;
"swtotal_size": number;
"swtotal_weight": number;
"swtxs": number;
"time": number;
"total_out": number;
"total_size": number;
"total_weight": number;
"totalfee": number;
"txs": number;
"utxo_increase": number;
"utxo_size_inc": number;
}
} }

2
backend/src/api/bitcoin/bitcoin-api.ts
View file

@ -28,7 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi {
size: block.size, size: block.size,
weight: block.weight, weight: block.weight,
previousblockhash: block.previousblockhash, previousblockhash: block.previousblockhash,
medianTime: block.mediantime, mediantime: block.mediantime,
}; };
} }

2
backend/src/api/bitcoin/esplora-api.interface.ts
View file

@ -88,7 +88,7 @@ export namespace IEsploraApi {
size: number; size: number;
weight: number; weight: number;
previousblockhash: string; previousblockhash: string;
medianTime?: number; mediantime: number;
} }
export interface Address { export interface Address {

250
backend/src/api/blocks.ts
View file

@ -2,7 +2,7 @@ import config from '../config';
import bitcoinApi from './bitcoin/bitcoin-api-factory'; import bitcoinApi from './bitcoin/bitcoin-api-factory';
import logger from '../logger'; import logger from '../logger';
import memPool from './mempool'; import memPool from './mempool';
import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces'; import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
import { Common } from './common'; import { Common } from './common';
import diskCache from './disk-cache'; import diskCache from './disk-cache';
import transactionUtils from './transaction-utils'; import transactionUtils from './transaction-utils';
@ -13,7 +13,6 @@ import poolsRepository from '../repositories/PoolsRepository';
import blocksRepository from '../repositories/BlocksRepository'; import blocksRepository from '../repositories/BlocksRepository';
import loadingIndicators from './loading-indicators'; import loadingIndicators from './loading-indicators';
import BitcoinApi from './bitcoin/bitcoin-api'; import BitcoinApi from './bitcoin/bitcoin-api';
import { prepareBlock } from '../utils/blocks-utils';
import BlocksRepository from '../repositories/BlocksRepository'; import BlocksRepository from '../repositories/BlocksRepository';
import HashratesRepository from '../repositories/HashratesRepository'; import HashratesRepository from '../repositories/HashratesRepository';
import indexer from '../indexer'; import indexer from '../indexer';
@ -143,7 +142,7 @@ class Blocks {
* @param block * @param block
* @returns BlockSummary * @returns BlockSummary
*/ */
private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary { public summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
const stripped = block.tx.map((tx) => { const stripped = block.tx.map((tx) => {
return { return {
txid: tx.txid, txid: tx.txid,
@ -166,80 +165,81 @@ class Blocks {
* @returns BlockExtended * @returns BlockExtended
*/ */
private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> { private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
const blk: BlockExtended = Object.assign({ extras: {} }, block); const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); const blk: Partial<BlockExtended> = Object.assign({}, block);
blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig; const extras: Partial<BlockExtension> = {};
blk.extras.usd = priceUpdater.latestPrices.USD;
blk.extras.medianTimestamp = block.medianTime; extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blk.extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height); extras.coinbaseRaw = coinbaseTx.vin[0].scriptsig;
extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
if (block.height === 0) { if (block.height === 0) {
blk.extras.medianFee = 0; // 50th percentiles extras.medianFee = 0; // 50th percentiles
blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
blk.extras.totalFees = 0; extras.totalFees = 0;
blk.extras.avgFee = 0; extras.avgFee = 0;
blk.extras.avgFeeRate = 0; extras.avgFeeRate = 0;
blk.extras.utxoSetChange = 0; extras.utxoSetChange = 0;
blk.extras.avgTxSize = 0; extras.avgTxSize = 0;
blk.extras.totalInputs = 0; extras.totalInputs = 0;
blk.extras.totalOutputs = 1; extras.totalOutputs = 1;
blk.extras.totalOutputAmt = 0; extras.totalOutputAmt = 0;
blk.extras.segwitTotalTxs = 0; extras.segwitTotalTxs = 0;
blk.extras.segwitTotalSize = 0; extras.segwitTotalSize = 0;
blk.extras.segwitTotalWeight = 0; extras.segwitTotalWeight = 0;
} else { } else {
const stats = await bitcoinClient.getBlockStats(block.id); const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
blk.extras.totalFees = stats.totalfee; extras.totalFees = stats.totalfee;
blk.extras.avgFee = stats.avgfee; extras.avgFee = stats.avgfee;
blk.extras.avgFeeRate = stats.avgfeerate; extras.avgFeeRate = stats.avgfeerate;
blk.extras.utxoSetChange = stats.utxo_increase; extras.utxoSetChange = stats.utxo_increase;
blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01; extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
blk.extras.totalInputs = stats.ins; extras.totalInputs = stats.ins;
blk.extras.totalOutputs = stats.outs; extras.totalOutputs = stats.outs;
blk.extras.totalOutputAmt = stats.total_out; extras.totalOutputAmt = stats.total_out;
blk.extras.segwitTotalTxs = stats.swtxs; extras.segwitTotalTxs = stats.swtxs;
blk.extras.segwitTotalSize = stats.swtotal_size; extras.segwitTotalSize = stats.swtotal_size;
blk.extras.segwitTotalWeight = stats.swtotal_weight; extras.segwitTotalWeight = stats.swtotal_weight;
} }
if (Common.blocksSummariesIndexingEnabled()) { if (Common.blocksSummariesIndexingEnabled()) {
blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
if (blk.extras.feePercentiles !== null) { if (extras.feePercentiles !== null) {
blk.extras.medianFeeAmt = blk.extras.feePercentiles[3]; extras.medianFeeAmt = extras.feePercentiles[3];
} }
} }
blk.extras.virtualSize = block.weight / 4.0; extras.virtualSize = block.weight / 4.0;
if (blk.extras.coinbaseTx.vout.length > 0) { if (coinbaseTx?.vout.length > 0) {
blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null; extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null; extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null;
blk.extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(blk.extras.coinbaseTx.vin[0].scriptsig) ?? null; extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null;
} else { } else {
blk.extras.coinbaseAddress = null; extras.coinbaseAddress = null;
blk.extras.coinbaseSignature = null; extras.coinbaseSignature = null;
blk.extras.coinbaseSignatureAscii = null; extras.coinbaseSignatureAscii = null;
} }
const header = await bitcoinClient.getBlockHeader(block.id, false); const header = await bitcoinClient.getBlockHeader(block.id, false);
blk.extras.header = header; extras.header = header;
const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex'); const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) { if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height); const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
blk.extras.utxoSetSize = txoutset.txouts, extras.utxoSetSize = txoutset.txouts,
blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000); extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
} else { } else {
blk.extras.utxoSetSize = null; extras.utxoSetSize = null;
blk.extras.totalInputAmt = null; extras.totalInputAmt = null;
} }
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
let pool: PoolTag; let pool: PoolTag;
if (blk.extras?.coinbaseTx !== undefined) { if (coinbaseTx !== undefined) {
pool = await this.$findBlockMiner(blk.extras?.coinbaseTx); pool = await this.$findBlockMiner(coinbaseTx);
} else { } else {
if (config.DATABASE.ENABLED === true) { if (config.DATABASE.ENABLED === true) {
pool = await poolsRepository.$getUnknownPool(); pool = await poolsRepository.$getUnknownPool();
@ -252,22 +252,24 @@ class Blocks {
logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` + logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
`Check your "pools" table entries`); `Check your "pools" table entries`);
} else { } else {
blk.extras.pool = { extras.pool = {
id: pool.id, id: pool.uniqueId,
name: pool.name, name: pool.name,
slug: pool.slug, slug: pool.slug,
}; };
} }
extras.matchRate = null;
if (config.MEMPOOL.AUDIT) { if (config.MEMPOOL.AUDIT) {
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id); const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
if (auditScore != null) { if (auditScore != null) {
blk.extras.matchRate = auditScore.matchRate; extras.matchRate = auditScore.matchRate;
} }
} }
} }
return blk; blk.extras = <BlockExtension>extras;
return <BlockExtended>blk;
} }
/** /**
@ -293,15 +295,18 @@ class Blocks {
} else { } else {
pools = poolsParser.miningPools; pools = poolsParser.miningPools;
} }
for (let i = 0; i < pools.length; ++i) { for (let i = 0; i < pools.length; ++i) {
if (address !== undefined) { if (address !== undefined) {
const addresses: string[] = JSON.parse(pools[i].addresses); const addresses: string[] = typeof pools[i].addresses === 'string' ?
JSON.parse(pools[i].addresses) : pools[i].addresses;
if (addresses.indexOf(address) !== -1) { if (addresses.indexOf(address) !== -1) {
return pools[i]; return pools[i];
} }
} }
const regexes: string[] = JSON.parse(pools[i].regexes); const regexes: string[] = typeof pools[i].regexes === 'string' ?
JSON.parse(pools[i].regexes) : pools[i].regexes;
for (let y = 0; y < regexes.length; ++y) { for (let y = 0; y < regexes.length; ++y) {
const regex = new RegExp(regexes[y], 'i'); const regex = new RegExp(regexes[y], 'i');
const match = asciiScriptSig.match(regex); const match = asciiScriptSig.match(regex);
@ -479,7 +484,7 @@ class Blocks {
loadingIndicators.setProgress('block-indexing', progress, false); loadingIndicators.setProgress('block-indexing', progress, false);
} }
const blockHash = await bitcoinApi.$getBlockHash(blockHeight); const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
const blockExtended = await this.$getBlockExtended(block, transactions); const blockExtended = await this.$getBlockExtended(block, transactions);
@ -527,13 +532,13 @@ class Blocks {
if (blockchainInfo.blocks === blockchainInfo.headers) { if (blockchainInfo.blocks === blockchainInfo.headers) {
const heightDiff = blockHeightTip % 2016; const heightDiff = blockHeightTip % 2016;
const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
this.lastDifficultyAdjustmentTime = block.timestamp; this.lastDifficultyAdjustmentTime = block.timestamp;
this.currentDifficulty = block.difficulty; this.currentDifficulty = block.difficulty;
if (blockHeightTip >= 2016) { if (blockHeightTip >= 2016) {
const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash) const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash);
this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
logger.debug(`Initial difficulty adjustment data set.`); logger.debug(`Initial difficulty adjustment data set.`);
} }
@ -565,18 +570,18 @@ class Blocks {
if (Common.indexingEnabled()) { if (Common.indexingEnabled()) {
if (!fastForwarded) { if (!fastForwarded) {
const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1); const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) { if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`); logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`);
// We assume there won't be a reorg with more than 10 block depth // We assume there won't be a reorg with more than 10 block depth
await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10); await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
await HashratesRepository.$deleteLastEntries(); await HashratesRepository.$deleteLastEntries();
await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10); await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10); await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
for (let i = 10; i >= 0; --i) { for (let i = 10; i >= 0; --i) {
const newBlock = await this.$indexBlock(lastBlock['height'] - i); const newBlock = await this.$indexBlock(lastBlock.height - i);
await this.$getStrippedBlockTransactions(newBlock.id, true, true); await this.$getStrippedBlockTransactions(newBlock.id, true, true);
if (config.MEMPOOL.CPFP_INDEXING) { if (config.MEMPOOL.CPFP_INDEXING) {
await this.$indexCPFP(newBlock.id, lastBlock['height'] - i); await this.$indexCPFP(newBlock.id, lastBlock.height - i);
} }
} }
await mining.$indexDifficultyAdjustments(); await mining.$indexDifficultyAdjustments();
@ -652,12 +657,12 @@ class Blocks {
if (Common.indexingEnabled()) { if (Common.indexingEnabled()) {
const dbBlock = await blocksRepository.$getBlockByHeight(height); const dbBlock = await blocksRepository.$getBlockByHeight(height);
if (dbBlock !== null) { if (dbBlock !== null) {
return prepareBlock(dbBlock); return dbBlock;
} }
} }
const blockHash = await bitcoinApi.$getBlockHash(height); const blockHash = await bitcoinApi.$getBlockHash(height);
const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions); const blockExtended = await this.$getBlockExtended(block, transactions);
@ -665,11 +670,11 @@ class Blocks {
await blocksRepository.$saveBlockInDatabase(blockExtended); await blocksRepository.$saveBlockInDatabase(blockExtended);
} }
return prepareBlock(blockExtended); return blockExtended;
} }
/** /**
* Index a block by hash if it's missing from the database. Returns the block after indexing * Get one block by its hash
*/ */
public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> { public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
// Check the memory cache // Check the memory cache
@ -678,31 +683,14 @@ class Blocks {
return blockByHash; return blockByHash;
} }
// Block has already been indexed // Not Bitcoin network, return the block as it from the bitcoin backend
if (Common.indexingEnabled()) {
const dbBlock = await blocksRepository.$getBlockByHash(hash);
if (dbBlock != null) {
return prepareBlock(dbBlock);
}
}
// Not Bitcoin network, return the block as it
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return await bitcoinApi.$getBlock(hash); return await bitcoinApi.$getBlock(hash);
} }
let block = await bitcoinClient.getBlock(hash);
block = prepareBlock(block);
// Bitcoin network, add our custom data on top // Bitcoin network, add our custom data on top
const transactions = await this.$getTransactionsExtended(hash, block.height, true); const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
const blockExtended = await this.$getBlockExtended(block, transactions); return await this.$indexBlock(block.height);
if (Common.indexingEnabled()) {
delete(blockExtended['coinbaseTx']);
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
return blockExtended;
} }
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false, public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
@ -736,6 +724,18 @@ class Blocks {
return summary.transactions; return summary.transactions;
} }
/**
* Get 15 blocks
*
* Internally this function uses two methods to get the blocks, and
* the method is automatically selected:
* - Using previous block hash links
* - Using block height
*
* @param fromHeight
* @param limit
* @returns
*/
public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> { public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight; let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
if (currentHeight > this.currentBlockHeight) { if (currentHeight > this.currentBlockHeight) {
@ -761,11 +761,14 @@ class Blocks {
for (let i = 0; i < limit && currentHeight >= 0; i++) { for (let i = 0; i < limit && currentHeight >= 0; i++) {
let block = this.getBlocks().find((b) => b.height === currentHeight); let block = this.getBlocks().find((b) => b.height === currentHeight);
if (block) { if (block) {
// Using the memory cache (find by height)
returnBlocks.push(block); returnBlocks.push(block);
} else if (Common.indexingEnabled()) { } else if (Common.indexingEnabled()) {
// Using indexing (find by height, index on the fly, save in database)
block = await this.$indexBlock(currentHeight); block = await this.$indexBlock(currentHeight);
returnBlocks.push(block); returnBlocks.push(block);
} else if (nextHash != null) { } else if (nextHash !== null) {
// Without indexing, query block on the fly using bitoin backend, follow previous hash links
block = await this.$indexBlock(currentHeight); block = await this.$indexBlock(currentHeight);
nextHash = block.previousblockhash; nextHash = block.previousblockhash;
returnBlocks.push(block); returnBlocks.push(block);
@ -790,7 +793,7 @@ class Blocks {
const blocks: any[] = []; const blocks: any[] = [];
while (fromHeight <= toHeight) { while (fromHeight <= toHeight) {
let block: any = await blocksRepository.$getBlockByHeight(fromHeight); let block: BlockExtended | null = await blocksRepository.$getBlockByHeight(fromHeight);
if (!block) { if (!block) {
await this.$indexBlock(fromHeight); await this.$indexBlock(fromHeight);
block = await blocksRepository.$getBlockByHeight(fromHeight); block = await blocksRepository.$getBlockByHeight(fromHeight);
@ -803,11 +806,11 @@ class Blocks {
const cleanBlock: any = { const cleanBlock: any = {
height: block.height ?? null, height: block.height ?? null,
hash: block.id ?? null, hash: block.id ?? null,
timestamp: block.blockTimestamp ?? null, timestamp: block.timestamp ?? null,
median_timestamp: block.medianTime ?? null, median_timestamp: block.mediantime ?? null,
previous_block_hash: block.previousblockhash ?? null, previous_block_hash: block.previousblockhash ?? null,
difficulty: block.difficulty ?? null, difficulty: block.difficulty ?? null,
header: block.header ?? null, header: block.extras.header ?? null,
version: block.version ?? null, version: block.version ?? null,
bits: block.bits ?? null, bits: block.bits ?? null,
nonce: block.nonce ?? null, nonce: block.nonce ?? null,
@ -815,29 +818,30 @@ class Blocks {
weight: block.weight ?? null, weight: block.weight ?? null,
tx_count: block.tx_count ?? null, tx_count: block.tx_count ?? null,
merkle_root: block.merkle_root ?? null, merkle_root: block.merkle_root ?? null,
reward: block.reward ?? null, reward: block.extras.reward ?? null,
total_fee_amt: block.fees ?? null, total_fee_amt: block.extras.totalFees ?? null,
avg_fee_amt: block.avg_fee ?? null, avg_fee_amt: block.extras.avgFee ?? null,
median_fee_amt: block.median_fee_amt ?? null, median_fee_amt: block.extras.medianFeeAmt ?? null,
fee_amt_percentiles: block.fee_percentiles ?? null, fee_amt_percentiles: block.extras.feePercentiles ?? null,
avg_fee_rate: block.avg_fee_rate ?? null, avg_fee_rate: block.extras.avgFeeRate ?? null,
median_fee_rate: block.median_fee ?? null, median_fee_rate: block.extras.medianFee ?? null,
fee_rate_percentiles: block.fee_span ?? null, fee_rate_percentiles: block.extras.feeRange ?? null,
total_inputs: block.total_inputs ?? null, total_inputs: block.extras.totalInputs ?? null,
total_input_amt: block.total_input_amt ?? null, total_input_amt: block.extras.totalInputAmt ?? null,
total_outputs: block.total_outputs ?? null, total_outputs: block.extras.totalOutputs ?? null,
total_output_amt: block.total_output_amt ?? null, total_output_amt: block.extras.totalOutputAmt ?? null,
segwit_total_txs: block.segwit_total_txs ?? null, segwit_total_txs: block.extras.segwitTotalTxs ?? null,
segwit_total_size: block.segwit_total_size ?? null, segwit_total_size: block.extras.segwitTotalSize ?? null,
segwit_total_weight: block.segwit_total_weight ?? null, segwit_total_weight: block.extras.segwitTotalWeight ?? null,
avg_tx_size: block.avg_tx_size ?? null, avg_tx_size: block.extras.avgTxSize ?? null,
utxoset_change: block.utxoset_change ?? null, utxoset_change: block.extras.utxoSetChange ?? null,
utxoset_size: block.utxoset_size ?? null, utxoset_size: block.extras.utxoSetSize ?? null,
coinbase_raw: block.coinbase_raw ?? null, coinbase_raw: block.extras.coinbaseRaw ?? null,
coinbase_address: block.coinbase_address ?? null, coinbase_address: block.extras.coinbaseAddress ?? null,
coinbase_signature: block.coinbase_signature ?? null, coinbase_signature: block.extras.coinbaseSignature ?? null,
coinbase_signature_ascii: block.coinbase_signature_ascii ?? null, coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null,
pool_slug: block.pool_slug ?? null, pool_slug: block.extras.pool.slug ?? null,
pool_id: block.extras.pool.id ?? null,
}; };
if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) { if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {

10
backend/src/api/chain-tips.ts
View file

@ -1,5 +1,5 @@
import logger from "../logger"; import logger from '../logger';
import bitcoinClient from "./bitcoin/bitcoin-client"; import bitcoinClient from './bitcoin/bitcoin-client';
export interface ChainTip { export interface ChainTip {
height: number; height: number;
@ -43,7 +43,11 @@ class ChainTips {
} }
} }
public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] { public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] {
if (height === undefined) {
return [];
}
const orphans: OrphanedBlock[] = []; const orphans: OrphanedBlock[] = [];
for (const block of this.orphanedBlocks) { for (const block of this.orphanedBlocks) {
if (block.height === height) { if (block.height === height) {

2
backend/src/api/disk-cache.ts
View file

@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces';
import { Common } from './common'; import { Common } from './common';
class DiskCache { class DiskCache {
private cacheSchemaVersion = 2; private cacheSchemaVersion = 3;
private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json'; private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json'; private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';

14
backend/src/api/mining/mining.ts
View file

@ -11,6 +11,8 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust
import config from '../../config'; import config from '../../config';
import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
import PricesRepository from '../../repositories/PricesRepository'; import PricesRepository from '../../repositories/PricesRepository';
import bitcoinApiFactory from '../bitcoin/bitcoin-api-factory';
import { IEsploraApi } from '../bitcoin/esplora-api.interface';
class Mining { class Mining {
blocksPriceIndexingRunning = false; blocksPriceIndexingRunning = false;
@ -189,8 +191,8 @@ class Mining {
try { try {
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisTimestamp = genesisBlock.time * 1000; const genesisTimestamp = genesisBlock.timestamp * 1000;
const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
const hashrates: any[] = []; const hashrates: any[] = [];
@ -292,8 +294,8 @@ class Mining {
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
try { try {
const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisTimestamp = genesisBlock.time * 1000; const genesisTimestamp = genesisBlock.timestamp * 1000;
const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
const lastMidnight = this.getDateMidnight(new Date()); const lastMidnight = this.getDateMidnight(new Date());
let toTimestamp = Math.round(lastMidnight.getTime()); let toTimestamp = Math.round(lastMidnight.getTime());
@ -394,13 +396,13 @@ class Mining {
} }
const blocks: any = await BlocksRepository.$getBlocksDifficulty(); const blocks: any = await BlocksRepository.$getBlocksDifficulty();
const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
let currentDifficulty = genesisBlock.difficulty; let currentDifficulty = genesisBlock.difficulty;
let totalIndexed = 0; let totalIndexed = 0;
if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) { if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
await DifficultyAdjustmentsRepository.$saveAdjustments({ await DifficultyAdjustmentsRepository.$saveAdjustments({
time: genesisBlock.time, time: genesisBlock.timestamp,
height: 0, height: 0,
difficulty: currentDifficulty, difficulty: currentDifficulty,
adjustment: 0.0, adjustment: 0.0,

2
backend/src/api/pools-parser.ts
View file

@ -7,6 +7,7 @@ import { PoolTag } from '../mempool.interfaces';
class PoolsParser { class PoolsParser {
miningPools: any[] = []; miningPools: any[] = [];
unknownPool: any = { unknownPool: any = {
'id': 0,
'name': 'Unknown', 'name': 'Unknown',
'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction', 'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
'regexes': '[]', 'regexes': '[]',
@ -26,6 +27,7 @@ class PoolsParser {
public setMiningPools(pools): void { public setMiningPools(pools): void {
for (const pool of pools) { for (const pool of pools) {
pool.regexes = pool.tags; pool.regexes = pool.tags;
pool.slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
delete(pool.tags); delete(pool.tags);
} }
this.miningPools = pools; this.miningPools = pools;

9
backend/src/index.ts
View file

@ -179,7 +179,14 @@ class Server {
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
this.currentBackendRetryInterval = 5; this.currentBackendRetryInterval = 5;
} catch (e: any) { } catch (e: any) {
const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`; let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
if (e?.stack) {
loggerMsg += ` Stack trace: ${e.stack}`;
}
// When we get a first Exception, only `logger.debug` it and retry after 5 seconds
// From the second Exception, `logger.warn` the Exception and increase the retry delay
// Maximum retry delay is 60 seconds
if (this.currentBackendRetryInterval > 5) { if (this.currentBackendRetryInterval > 5) {
logger.warn(loggerMsg); logger.warn(loggerMsg);
mempool.setOutOfSync(); mempool.setOutOfSync();

69
backend/src/mempool.interfaces.ts
View file

@ -1,9 +1,10 @@
import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
import { OrphanedBlock } from './api/chain-tips'; import { OrphanedBlock } from './api/chain-tips';
import { HeapNode } from "./utils/pairing-heap"; import { HeapNode } from './utils/pairing-heap';
export interface PoolTag { export interface PoolTag {
id: number; // mysql row id id: number;
uniqueId: number;
name: string; name: string;
link: string; link: string;
regexes: string; // JSON array regexes: string; // JSON array
@ -147,44 +148,44 @@ export interface TransactionStripped {
} }
export interface BlockExtension { export interface BlockExtension {
totalFees?: number; totalFees: number;
medianFee?: number; medianFee: number; // median fee rate
feeRange?: number[]; feeRange: number[]; // fee rate percentiles
reward?: number; reward: number;
coinbaseTx?: TransactionMinerInfo; matchRate: number | null;
matchRate?: number; pool: {
pool?: { id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id`
id: number;
name: string; name: string;
slug: string; slug: string;
}; };
avgFee?: number; avgFee: number;
avgFeeRate?: number; avgFeeRate: number;
coinbaseRaw?: string; coinbaseRaw: string;
usd?: number | null; orphans: OrphanedBlock[] | null;
medianTimestamp?: number; coinbaseAddress: string | null;
blockTime?: number; coinbaseSignature: string | null;
orphans?: OrphanedBlock[] | null; coinbaseSignatureAscii: string | null;
coinbaseAddress?: string | null; virtualSize: number;
coinbaseSignature?: string | null; avgTxSize: number;
coinbaseSignatureAscii?: string | null; totalInputs: number;
virtualSize?: number; totalOutputs: number;
avgTxSize?: number; totalOutputAmt: number;
totalInputs?: number; medianFeeAmt: number | null; // median fee in sats
totalOutputs?: number; feePercentiles: number[] | null, // fee percentiles in sats
totalOutputAmt?: number; segwitTotalTxs: number;
medianFeeAmt?: number | null; segwitTotalSize: number;
feePercentiles?: number[] | null, segwitTotalWeight: number;
segwitTotalTxs?: number; header: string;
segwitTotalSize?: number; utxoSetChange: number;
segwitTotalWeight?: number;
header?: string;
utxoSetChange?: number;
// Requires coinstatsindex, will be set to NULL otherwise // Requires coinstatsindex, will be set to NULL otherwise
utxoSetSize?: number | null; utxoSetSize: number | null;
totalInputAmt?: number | null; totalInputAmt: number | null;
} }
/**
* Note: Everything that is added in here will be automatically returned through
* /api/v1/block and /api/v1/blocks APIs
*/
export interface BlockExtended extends IEsploraApi.Block { export interface BlockExtended extends IEsploraApi.Block {
extras: BlockExtension; extras: BlockExtension;
} }

227
backend/src/repositories/BlocksRepository.ts
View file

@ -1,8 +1,7 @@
import { BlockExtended, BlockPrice } from '../mempool.interfaces'; import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces';
import DB from '../database'; import DB from '../database';
import logger from '../logger'; import logger from '../logger';
import { Common } from '../api/common'; import { Common } from '../api/common';
import { prepareBlock } from '../utils/blocks-utils';
import PoolsRepository from './PoolsRepository'; import PoolsRepository from './PoolsRepository';
import HashratesRepository from './HashratesRepository'; import HashratesRepository from './HashratesRepository';
import { escape } from 'mysql2'; import { escape } from 'mysql2';
@ -10,6 +9,51 @@ import BlocksSummariesRepository from './BlocksSummariesRepository';
import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository'; import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
import bitcoinClient from '../api/bitcoin/bitcoin-client'; import bitcoinClient from '../api/bitcoin/bitcoin-client';
import config from '../config'; import config from '../config';
import chainTips from '../api/chain-tips';
import blocks from '../api/blocks';
import BlocksAuditsRepository from './BlocksAuditsRepository';
const BLOCK_DB_FIELDS = `
blocks.hash AS id,
blocks.height,
blocks.version,
UNIX_TIMESTAMP(blocks.blockTimestamp) AS timestamp,
blocks.bits,
blocks.nonce,
blocks.difficulty,
blocks.merkle_root,
blocks.tx_count,
blocks.size,
blocks.weight,
blocks.previous_block_hash AS previousblockhash,
UNIX_TIMESTAMP(blocks.median_timestamp) AS mediantime,
blocks.fees AS totalFees,
blocks.median_fee AS medianFee,
blocks.fee_span AS feeRange,
blocks.reward,
pools.unique_id AS poolId,
pools.name AS poolName,
pools.slug AS poolSlug,
blocks.avg_fee AS avgFee,
blocks.avg_fee_rate AS avgFeeRate,
blocks.coinbase_raw AS coinbaseRaw,
blocks.coinbase_address AS coinbaseAddress,
blocks.coinbase_signature AS coinbaseSignature,
blocks.coinbase_signature_ascii AS coinbaseSignatureAscii,
blocks.avg_tx_size AS avgTxSize,
blocks.total_inputs AS totalInputs,
blocks.total_outputs AS totalOutputs,
blocks.total_output_amt AS totalOutputAmt,
blocks.median_fee_amt AS medianFeeAmt,
blocks.fee_percentiles AS feePercentiles,
blocks.segwit_total_txs AS segwitTotalTxs,
blocks.segwit_total_size AS segwitTotalSize,
blocks.segwit_total_weight AS segwitTotalWeight,
blocks.header,
blocks.utxoset_change AS utxoSetChange,
blocks.utxoset_size AS utxoSetSize,
blocks.total_input_amt AS totalInputAmts
`;
class BlocksRepository { class BlocksRepository {
/** /**
@ -44,6 +88,11 @@ class BlocksRepository {
?, ? ?, ?
)`; )`;
const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id);
if (!poolDbId) {
throw Error(`Could not find a mining pool with the unique_id = ${block.extras.pool.id}. This error should never be printed.`);
}
const params: any[] = [ const params: any[] = [
block.height, block.height,
block.id, block.id,
@ -53,7 +102,7 @@ class BlocksRepository {
block.tx_count, block.tx_count,
block.extras.coinbaseRaw, block.extras.coinbaseRaw,
block.difficulty, block.difficulty,
block.extras.pool?.id, // Should always be set to something poolDbId.id,
block.extras.totalFees, block.extras.totalFees,
JSON.stringify(block.extras.feeRange), JSON.stringify(block.extras.feeRange),
block.extras.medianFee, block.extras.medianFee,
@ -65,7 +114,7 @@ class BlocksRepository {
block.previousblockhash, block.previousblockhash,
block.extras.avgFee, block.extras.avgFee,
block.extras.avgFeeRate, block.extras.avgFeeRate,
block.extras.medianTimestamp, block.mediantime,
block.extras.header, block.extras.header,
block.extras.coinbaseAddress, block.extras.coinbaseAddress,
truncatedCoinbaseSignature, truncatedCoinbaseSignature,
@ -87,9 +136,9 @@ class BlocksRepository {
await DB.query(query, params); await DB.query(query, params);
} catch (e: any) { } catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`); logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`, logger.tags.mining);
} else { } else {
logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e)); logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
throw e; throw e;
} }
} }
@ -307,34 +356,17 @@ class BlocksRepository {
/** /**
* Get blocks mined by a specific mining pool * Get blocks mined by a specific mining pool
*/ */
public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> { public async $getBlocksByPool(slug: string, startHeight?: number): Promise<BlockExtended[]> {
const pool = await PoolsRepository.$getPool(slug); const pool = await PoolsRepository.$getPool(slug);
if (!pool) { if (!pool) {
throw new Error('This mining pool does not exist ' + escape(slug)); throw new Error('This mining pool does not exist ' + escape(slug));
} }
const params: any[] = []; const params: any[] = [];
let query = ` SELECT let query = `
blocks.height, SELECT ${BLOCK_DB_FIELDS}
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
size,
weight,
tx_count,
coinbase_raw,
difficulty,
fees,
fee_span,
median_fee,
reward,
version,
bits,
nonce,
merkle_root,
previous_block_hash as previousblockhash,
avg_fee,
avg_fee_rate
FROM blocks FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE pool_id = ?`; WHERE pool_id = ?`;
params.push(pool.id); params.push(pool.id);
@ -347,11 +379,11 @@ class BlocksRepository {
LIMIT 10`; LIMIT 10`;
try { try {
const [rows] = await DB.query(query, params); const [rows]: any[] = await DB.query(query, params);
const blocks: BlockExtended[] = []; const blocks: BlockExtended[] = [];
for (const block of <object[]>rows) { for (const block of rows) {
blocks.push(prepareBlock(block)); blocks.push(await this.formatDbBlockIntoExtendedBlock(block));
} }
return blocks; return blocks;
@ -364,32 +396,21 @@ class BlocksRepository {
/** /**
* Get one block by height * Get one block by height
*/ */
public async $getBlockByHeight(height: number): Promise<object | null> { public async $getBlockByHeight(height: number): Promise<BlockExtended | null> {
try { try {
const [rows]: any[] = await DB.query(`SELECT const [rows]: any[] = await DB.query(`
blocks.*, SELECT ${BLOCK_DB_FIELDS}
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
UNIX_TIMESTAMP(blocks.median_timestamp) as medianTime,
pools.id as pool_id,
pools.name as pool_name,
pools.link as pool_link,
pools.slug as pool_slug,
pools.addresses as pool_addresses,
pools.regexes as pool_regexes,
previous_block_hash as previousblockhash
FROM blocks FROM blocks
JOIN pools ON blocks.pool_id = pools.id JOIN pools ON blocks.pool_id = pools.id
WHERE blocks.height = ${height} WHERE blocks.height = ?`,
`); [height]
);
if (rows.length <= 0) { if (rows.length <= 0) {
return null; return null;
} }
rows[0].fee_span = JSON.parse(rows[0].fee_span); return await this.formatDbBlockIntoExtendedBlock(rows[0]);
rows[0].fee_percentiles = JSON.parse(rows[0].fee_percentiles);
return rows[0];
} catch (e) { } catch (e) {
logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e)); logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
throw e; throw e;
@ -402,10 +423,7 @@ class BlocksRepository {
public async $getBlockByHash(hash: string): Promise<object | null> { public async $getBlockByHash(hash: string): Promise<object | null> {
try { try {
const query = ` const query = `
SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id, SELECT ${BLOCK_DB_FIELDS}
pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
pools.addresses as pool_addresses, pools.regexes as pool_regexes,
previous_block_hash as previousblockhash
FROM blocks FROM blocks
JOIN pools ON blocks.pool_id = pools.id JOIN pools ON blocks.pool_id = pools.id
WHERE hash = ?; WHERE hash = ?;
@ -415,9 +433,8 @@ class BlocksRepository {
if (rows.length <= 0) { if (rows.length <= 0) {
return null; return null;
} }
rows[0].fee_span = JSON.parse(rows[0].fee_span); return await this.formatDbBlockIntoExtendedBlock(rows[0]);
return rows[0];
} catch (e) { } catch (e) {
logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e)); logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
throw e; throw e;
@ -508,8 +525,15 @@ class BlocksRepository {
public async $validateChain(): Promise<boolean> { public async $validateChain(): Promise<boolean> {
try { try {
const start = new Date().getTime(); const start = new Date().getTime();
const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash, const [blocks]: any[] = await DB.query(`
UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`); SELECT
height,
hash,
previous_block_hash,
UNIX_TIMESTAMP(blockTimestamp) AS timestamp
FROM blocks
ORDER BY height
`);
let partialMsg = false; let partialMsg = false;
let idx = 1; let idx = 1;
@ -833,6 +857,95 @@ class BlocksRepository {
throw e; throw e;
} }
} }
/**
* Convert a mysql row block into a BlockExtended. Note that you
* must provide the correct field into dbBlk object param
*
* @param dbBlk
*/
private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise<BlockExtended> {
const blk: Partial<BlockExtended> = {};
const extras: Partial<BlockExtension> = {};
// IEsploraApi.Block
blk.id = dbBlk.id;
blk.height = dbBlk.height;
blk.version = dbBlk.version;
blk.timestamp = dbBlk.timestamp;
blk.bits = dbBlk.bits;
blk.nonce = dbBlk.nonce;
blk.difficulty = dbBlk.difficulty;
blk.merkle_root = dbBlk.merkle_root;
blk.tx_count = dbBlk.tx_count;
blk.size = dbBlk.size;
blk.weight = dbBlk.weight;
blk.previousblockhash = dbBlk.previousblockhash;
blk.mediantime = dbBlk.mediantime;
// BlockExtension
extras.totalFees = dbBlk.totalFees;
extras.medianFee = dbBlk.medianFee;
extras.feeRange = JSON.parse(dbBlk.feeRange);
extras.reward = dbBlk.reward;
extras.pool = {
id: dbBlk.poolId,
name: dbBlk.poolName,
slug: dbBlk.poolSlug,
};
extras.avgFee = dbBlk.avgFee;
extras.avgFeeRate = dbBlk.avgFeeRate;
extras.coinbaseRaw = dbBlk.coinbaseRaw;
extras.coinbaseAddress = dbBlk.coinbaseAddress;
extras.coinbaseSignature = dbBlk.coinbaseSignature;
extras.coinbaseSignatureAscii = dbBlk.coinbaseSignatureAscii;
extras.avgTxSize = dbBlk.avgTxSize;
extras.totalInputs = dbBlk.totalInputs;
extras.totalOutputs = dbBlk.totalOutputs;
extras.totalOutputAmt = dbBlk.totalOutputAmt;
extras.medianFeeAmt = dbBlk.medianFeeAmt;
extras.feePercentiles = JSON.parse(dbBlk.feePercentiles);
extras.segwitTotalTxs = dbBlk.segwitTotalTxs;
extras.segwitTotalSize = dbBlk.segwitTotalSize;
extras.segwitTotalWeight = dbBlk.segwitTotalWeight;
extras.header = dbBlk.header,
extras.utxoSetChange = dbBlk.utxoSetChange;
extras.utxoSetSize = dbBlk.utxoSetSize;
extras.totalInputAmt = dbBlk.totalInputAmt;
extras.virtualSize = dbBlk.weight / 4.0;
// Re-org can happen after indexing so we need to always get the
// latest state from core
extras.orphans = chainTips.getOrphanedBlocksAtHeight(dbBlk.height);
// Match rate is not part of the blocks table, but it is part of APIs so we must include it
extras.matchRate = null;
if (config.MEMPOOL.AUDIT) {
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(dbBlk.id);
if (auditScore != null) {
extras.matchRate = auditScore.matchRate;
}
}
// If we're missing block summary related field, check if we can populate them on the fly now
if (Common.blocksSummariesIndexingEnabled() &&
(extras.medianFeeAmt === null || extras.feePercentiles === null))
{
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
if (extras.feePercentiles === null) {
const block = await bitcoinClient.getBlock(dbBlk.id, 2);
const summary = blocks.summarizeBlock(block);
await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
}
if (extras.feePercentiles !== null) {
extras.medianFeeAmt = extras.feePercentiles[3];
}
}
blk.extras = <BlockExtension>extras;
return <BlockExtended>blk;
}
} }
export default new BlocksRepository(); export default new BlocksRepository();

6
backend/src/repositories/PoolsRepository.ts
View file

@ -10,7 +10,7 @@ class PoolsRepository {
* Get all pools tagging info * Get all pools tagging info
*/ */
public async $getPools(): Promise<PoolTag[]> { public async $getPools(): Promise<PoolTag[]> {
const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;'); const [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, addresses, regexes, slug FROM pools');
return <PoolTag[]>rows; return <PoolTag[]>rows;
} }
@ -18,10 +18,10 @@ class PoolsRepository {
* Get unknown pool tagging info * Get unknown pool tagging info
*/ */
public async $getUnknownPool(): Promise<PoolTag> { public async $getUnknownPool(): Promise<PoolTag> {
let [rows]: any[] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"'); let [rows]: any[] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
if (rows && rows.length === 0 && config.DATABASE.ENABLED) { if (rows && rows.length === 0 && config.DATABASE.ENABLED) {
await poolsParser.$insertUnknownPool(); await poolsParser.$insertUnknownPool();
[rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"'); [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
} }
return <PoolTag>rows[0]; return <PoolTag>rows[0];
} }

33
backend/src/utils/blocks-utils.ts
View file

@ -1,33 +0,0 @@
import { BlockExtended } from '../mempool.interfaces';
export function prepareBlock(block: any): BlockExtended {
return <BlockExtended>{
id: block.id ?? block.hash, // hash for indexed block
timestamp: block.timestamp ?? block.time ?? block.blockTimestamp, // blockTimestamp for indexed block
height: block.height,
version: block.version,
bits: (typeof block.bits === 'string' ? parseInt(block.bits, 16): block.bits),
nonce: block.nonce,
difficulty: block.difficulty,
merkle_root: block.merkle_root ?? block.merkleroot,
tx_count: block.tx_count ?? block.nTx,
size: block.size,
weight: block.weight,
previousblockhash: block.previousblockhash,
extras: {
coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
feeRange: block.feeRange ?? block?.extras?.feeRange ?? block.fee_span,
reward: block.reward ?? block?.extras?.reward,
totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
avgFee: block?.extras?.avgFee ?? block.avg_fee,
avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate,
pool: block?.extras?.pool ?? (block?.pool_id ? {
id: block.pool_id,
name: block.pool_name,
slug: block.pool_slug,
} : undefined),
usd: block?.extras?.usd ?? block.usd ?? null,
}
};
}

1
frontend/src/app/interfaces/node-api.interface.ts
View file

@ -114,7 +114,6 @@ export interface BlockExtension {
medianFee?: number; medianFee?: number;
feeRange?: number[]; feeRange?: number[];
reward?: number; reward?: number;
coinbaseTx?: Transaction;
coinbaseRaw?: string; coinbaseRaw?: string;
matchRate?: number; matchRate?: number;
pool?: { pool?: {