diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 1043c344f..c9bb46754 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,11 +2,15 @@ import config from '../config'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { BlockExtended, TransactionExtended } from '../mempool.interfaces'; +import { BlockExtended, PoolTag, TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; import bitcoinClient from './bitcoin/bitcoin-client'; +import { DB } from '../database'; +import { IEsploraApi } from './bitcoin/esplora-api.interface'; +import poolsRepository from '../repositories/PoolsRepository'; +import blocksRepository from '../repositories/BlocksRepository'; class Blocks { private blocks: BlockExtended[] = []; @@ -30,6 +34,137 @@ class Blocks { this.newBlockCallbacks.push(fn); } + /** + * Return the list of transaction for a block + * @param blockHash + * @param blockHeight + * @param onlyCoinbase - Set to true if you only need the coinbase transaction + * @returns Promise + */ + private async $getTransactionsExtended(blockHash: string, blockHeight: number, onlyCoinbase: boolean) : Promise { + const transactions: TransactionExtended[] = []; + const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); + + const mempool = memPool.getMempool(); + let transactionsFound = 0; + let transactionsFetched = 0; + + for (let i = 0; i < txIds.length; i++) { + if (mempool[txIds[i]]) { + // We update blocks before the mempool (index.ts), therefore we can + // optimize here by directly fetching txs in the "outdated" mempool + transactions.push(mempool[txIds[i]]); + transactionsFound++; + } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) { + // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...) + if (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length) { // Avoid log spam + logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`); + } + try { + const tx = await transactionUtils.$getTransactionExtended(txIds[i]); + transactions.push(tx); + transactionsFetched++; + } catch (e) { + logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e)); + if (i === 0) { + throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]); + } + } + } + + if (onlyCoinbase === true) { + break; // Fetch the first transaction and exit + } + } + + transactions.forEach((tx) => { + if (!tx.cpfpChecked) { + Common.setRelativesAndGetCpfpInfo(tx, mempool); // Child Pay For Parent + } + }); + + logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`); + + return transactions; + } + + /** + * Return a block with additional data (reward, coinbase, fees...) + * @param block + * @param transactions + * @returns BlockExtended + */ + private getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]) : BlockExtended { + const blockExtended: BlockExtended = Object.assign({}, block); + blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); + blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); + + const transactionsTmp = [...transactions]; + transactionsTmp.shift(); + transactionsTmp.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize); + blockExtended.medianFee = transactionsTmp.length > 0 ? Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0; + blockExtended.feeRange = transactionsTmp.length > 0 ? Common.getFeesInRange(transactionsTmp, 8) : [0, 0]; + + return blockExtended; + } + + /** + * Try to find which miner found the block + * @param txMinerInfo + * @returns + */ + private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined) : Promise { + if (txMinerInfo === undefined) { + return poolsRepository.getUnknownPool(); + } + + const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig); + const address = txMinerInfo.vout[0].scriptpubkey_address; + + const pools: PoolTag[] = await poolsRepository.$getPools(); + for (let i = 0; i < pools.length; ++i) { + if (address !== undefined) { + let addresses: string[] = JSON.parse(pools[i].addresses); + if (addresses.indexOf(address) !== -1) { + return pools[i]; + } + } + + let regexes: string[] = JSON.parse(pools[i].regexes); + for (let y = 0; y < regexes.length; ++y) { + let match = asciiScriptSig.match(regexes[y]); + if (match !== null) { + return pools[i]; + } + } + } + + return poolsRepository.getUnknownPool(); + } + + /** + * Index all blocks metadata for the mining dashboard + */ + public async $generateBlockDatabase() { + let currentBlockHeight = await bitcoinApi.$getBlockHeightTip(); + let maxBlocks = 100; // tmp + + while (currentBlockHeight-- > 0 && maxBlocks-- > 0) { + if (await blocksRepository.$isBlockAlreadyIndexed(currentBlockHeight)) { + // logger.debug(`Block #${currentBlockHeight} already indexed, skipping`); + continue; + } + logger.debug(`Indexing block #${currentBlockHeight}`); + const blockHash = await bitcoinApi.$getBlockHash(currentBlockHeight); + const block = await bitcoinApi.$getBlock(blockHash); + const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); + const blockExtended = this.getBlockExtended(block, transactions); + const miner = await this.$findBlockMiner(blockExtended.coinbaseTx); + const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); + await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner); + } + } + public async $updateBlocks() { const blockHeightTip = await bitcoinApi.$getBlockHeightTip(); @@ -70,48 +205,14 @@ class Blocks { logger.debug(`New block found (#${this.currentBlockHeight})!`); } - const transactions: TransactionExtended[] = []; - const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight); const block = await bitcoinApi.$getBlock(blockHash); const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); - - const mempool = memPool.getMempool(); - let transactionsFound = 0; - - for (let i = 0; i < txIds.length; i++) { - if (mempool[txIds[i]]) { - transactions.push(mempool[txIds[i]]); - transactionsFound++; - } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) { - logger.debug(`Fetching block tx ${i} of ${txIds.length}`); - try { - const tx = await transactionUtils.$getTransactionExtended(txIds[i]); - transactions.push(tx); - } catch (e) { - logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e)); - if (i === 0) { - throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]); - } - } - } - } - - transactions.forEach((tx) => { - if (!tx.cpfpChecked) { - Common.setRelativesAndGetCpfpInfo(tx, mempool); - } - }); - - logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${txIds.length - transactionsFound} not found.`); - - const blockExtended: BlockExtended = Object.assign({}, block); - blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); - blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); - transactions.shift(); - transactions.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize); - blockExtended.medianFee = transactions.length > 0 ? Common.median(transactions.map((tx) => tx.effectiveFeePerVsize)) : 0; - blockExtended.feeRange = transactions.length > 0 ? Common.getFeesInRange(transactions, 8) : [0, 0]; + const transactions = await this.$getTransactionsExtended(blockHash, block.height, false); + const blockExtended: BlockExtended = this.getBlockExtended(block, transactions); + const miner = await this.$findBlockMiner(blockExtended.coinbaseTx); + const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); + await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner); if (block.height % 2016 === 0) { this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100; @@ -130,6 +231,8 @@ class Blocks { if (memPool.isInSync()) { diskCache.$saveCacheToDisk(); } + + return; } } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 4d6c35860..859912f6c 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -118,11 +118,11 @@ class Mempool { }); } hasChange = true; - if (diff > 0) { - logger.debug('Fetched transaction ' + txCount + ' / ' + diff); - } else { - logger.debug('Fetched transaction ' + txCount); - } + // if (diff > 0) { + // logger.debug('Fetched transaction ' + txCount + ' / ' + diff); + // } else { + // logger.debug('Fetched transaction ' + txCount); + // } newTransactions.push(transaction); } catch (e) { logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 1496b810b..2e669d709 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -44,6 +44,14 @@ class TransactionUtils { } return transactionExtended; } + + public hex2ascii(hex: string) { + let str = ''; + for (let i = 0; i < hex.length; i += 2) { + str += String.fromCharCode(parseInt(hex.substr(i, 2), 16)); + } + return str; + } } export default new TransactionUtils(); diff --git a/backend/src/index.ts b/backend/src/index.ts index 420c60365..be26d53fe 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -25,7 +25,6 @@ import databaseMigration from './api/database-migration'; import poolsParser from './api/pools-parser'; import syncAssets from './sync-assets'; import icons from './api/liquid/icons'; -import poolsParser from './api/pools-parser'; import { Common } from './api/common'; class Server { @@ -33,6 +32,7 @@ class Server { private server: http.Server | undefined; private app: Express; private currentBackendRetryInterval = 5; + private blockIndexingStarted = false; constructor() { this.app = express(); @@ -90,7 +90,6 @@ class Server { await checkDbConnection(); try { await databaseMigration.$initializeOrMigrateDatabase(); - await poolsParser.migratePoolsJson(); } catch (e) { throw new Error(e instanceof Error ? e.message : 'Error'); } @@ -139,6 +138,13 @@ class Server { } await blocks.$updateBlocks(); await memPool.$updateMempool(); + + if (this.blockIndexingStarted === false/* && memPool.isInSync()*/) { + blocks.$generateBlockDatabase(); + this.blockIndexingStarted = true; + logger.info("START OLDER BLOCK INDEXING"); + } + setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); this.currentBackendRetryInterval = 5; } catch (e) { diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 2604a233c..ae921f073 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -1,5 +1,13 @@ +import { RowDataPacket } from 'mysql2'; import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; +export interface PoolTag extends RowDataPacket { + name: string, + link: string, + regexes: string, + addresses: string, +} + export interface MempoolBlock { blockSize: number; blockVSize: number; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts new file mode 100644 index 000000000..f62b261a1 --- /dev/null +++ b/backend/src/repositories/BlocksRepository.ts @@ -0,0 +1,73 @@ +import { IEsploraApi } from "../api/bitcoin/esplora-api.interface"; +import { BlockExtended, PoolTag } from "../mempool.interfaces"; +import { DB } from "../database"; +import logger from "../logger"; +import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; + +class BlocksRepository { + /** + * Save indexed block data in the database + * @param block + * @param blockHash + * @param coinbaseTxid + * @param poolTag + */ + public async $saveBlockInDatabase( + block: BlockExtended, + blockHash: string, + coinbaseHex: string | undefined, + poolTag: PoolTag + ) { + const connection = await DB.pool.getConnection(); + + try { + const query = `INSERT INTO blocks( + height, hash, timestamp, size, + weight, tx_count, coinbase_raw, difficulty, + pool_id, fees, fee_span, median_fee + ) VALUE ( + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ? + )`; + + const params: any[] = [ + block.height, blockHash, block.timestamp, block.size, + block.weight, block.tx_count, coinbaseHex ? coinbaseHex : "", block.difficulty, + poolTag.id, 0, "[]", block.medianFee, + ]; + + await connection.query(query, params); + } catch (e) { + console.log(e); + logger.err('$updateBlocksDatabase() error' + (e instanceof Error ? e.message : e)); + } + + connection.release(); + } + + /** + * Check if a block has already been indexed in the database. Query the databse directly. + * This can be cached/optimized if required later on to avoid too many db queries. + * @param blockHeight + * @returns + */ + public async $isBlockAlreadyIndexed(blockHeight: number) { + const connection = await DB.pool.getConnection(); + let exists = false; + + try { + const query = `SELECT height from blocks where blocks.height = ${blockHeight}`; + const [rows]: any[] = await connection.query(query); + exists = rows.length === 1; + } catch (e) { + console.log(e); + logger.err('$isBlockAlreadyIndexed() error' + (e instanceof Error ? e.message : e)); + } + connection.release(); + + return exists; + } +} + +export default new BlocksRepository(); \ No newline at end of file diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts new file mode 100644 index 000000000..aa6f457f2 --- /dev/null +++ b/backend/src/repositories/PoolsRepository.ts @@ -0,0 +1,30 @@ +import { FieldPacket } from "mysql2"; +import { DB } from "../database"; +import { PoolTag } from "../mempool.interfaces" + +class PoolsRepository { + /** + * Get all pools tagging info + */ + public async $getPools() : Promise { + const connection = await DB.pool.getConnection(); + const [rows]: [PoolTag[], FieldPacket[]] = await connection.query("SELECT * FROM pools;"); + connection.release(); + return rows; + } + + /** + * Get unknown pool tagging info + */ + public getUnknownPool(): PoolTag { + return { + id: null, + name: 'Unknown', + link: 'rickroll?', + regexes: "[]", + addresses: "[]", + }; + } +} + +export default new PoolsRepository(); \ No newline at end of file