From 255919f03ff7e68df2880582e810428fca1d9e51 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 24 Jun 2024 07:29:02 +0000 Subject: [PATCH] Update pool instead of deleting blocks --- backend/src/api/blocks.ts | 26 +--- backend/src/api/pools-parser.ts | 143 +++++++++++-------- backend/src/repositories/BlocksRepository.ts | 23 ++- 3 files changed, 107 insertions(+), 85 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 53a2ce1ac..fc7dac135 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -372,8 +372,7 @@ class Blocks { } } - const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig); - const addresses = txMinerInfo.vout.map((vout) => vout.scriptpubkey_address).filter((address) => address); + const addresses = txMinerInfo.vout.map((vout) => vout.scriptpubkey_address).filter(address => address) as string[]; let pools: PoolTag[] = []; if (config.DATABASE.ENABLED === true) { @@ -382,26 +381,9 @@ class Blocks { pools = poolsParser.miningPools; } - for (let i = 0; i < pools.length; ++i) { - if (addresses.length) { - const poolAddresses: string[] = typeof pools[i].addresses === 'string' ? - JSON.parse(pools[i].addresses) : pools[i].addresses; - for (let y = 0; y < poolAddresses.length; y++) { - if (addresses.indexOf(poolAddresses[y]) !== -1) { - return pools[i]; - } - } - } - - const regexes: string[] = typeof pools[i].regexes === 'string' ? - JSON.parse(pools[i].regexes) : pools[i].regexes; - for (let y = 0; y < regexes.length; ++y) { - const regex = new RegExp(regexes[y], 'i'); - const match = asciiScriptSig.match(regex); - if (match !== null) { - return pools[i]; - } - } + const pool = poolsParser.matchBlockMiner(txMinerInfo.vin[0].scriptsig, addresses || [], pools); + if (pool) { + return pool; } if (config.DATABASE.ENABLED === true) { diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index e04565a01..c09f1ba1e 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -5,6 +5,8 @@ import PoolsRepository from '../repositories/PoolsRepository'; import { PoolTag } from '../mempool.interfaces'; import diskCache from './disk-cache'; import mining from './mining/mining'; +import transactionUtils from './transaction-utils'; +import BlocksRepository from '../repositories/BlocksRepository'; class PoolsParser { miningPools: any[] = []; @@ -37,7 +39,7 @@ class PoolsParser { /** * Populate our db with updated mining pool definition - * @param pools + * @param pools */ public async migratePoolsJson(): Promise { // We also need to wipe the backend cache to make sure we don't serve blocks with @@ -46,6 +48,8 @@ class PoolsParser { await this.$insertUnknownPool(); + let reindexUnknown = false; + for (const pool of this.miningPools) { if (!pool.id) { logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`); @@ -57,22 +61,22 @@ class PoolsParser { logger.err(`Mining pool ${pool.name} must have at least one of the fields 'addresses' or 'regexes'. Skipping.`); continue; } - + pool.addresses = pool.addresses || []; pool.regexes = pool.regexes || []; - + if (pool.addresses.length === 0 && pool.regexes.length === 0) { logger.err(`Mining pool ${pool.name} has no 'addresses' nor 'regexes' defined. Skipping.`); continue; } - + if (pool.addresses.length === 0) { logger.warn(`Mining pool ${pool.name} has no 'addresses' defined.`); } - + if (pool.regexes.length === 0) { logger.warn(`Mining pool ${pool.name} has no 'regexes' defined.`); - } + } const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false); if (!poolDB) { @@ -80,7 +84,7 @@ class PoolsParser { const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase(); logger.debug(`Inserting new mining pool ${pool.name}`); await PoolsRepository.$insertNewMiningPool(pool, slug); - await this.$deleteUnknownBlocks(); + reindexUnknown = true; } else { if (poolDB.name !== pool.name) { // Pool has been renamed @@ -98,7 +102,45 @@ class PoolsParser { // Pool addresses changed or coinbase tags changed logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool.`); await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes); - await this.$deleteBlocksForPool(poolDB); + reindexUnknown = true; + await this.$reindexBlocksForPool(poolDB.id); + } + } + } + + if (reindexUnknown) { + logger.notice(`Updating addresses and/or coinbase tags for unknown mining pool.`); + let unknownPool; + if (config.DATABASE.ENABLED === true) { + unknownPool = await PoolsRepository.$getUnknownPool(); + } else { + unknownPool = this.unknownPool; + } + await this.$reindexBlocksForPool(unknownPool.id); + } + } + + public matchBlockMiner(scriptsig: string, addresses: string[], pools: PoolTag[]): PoolTag | undefined { + const asciiScriptSig = transactionUtils.hex2ascii(scriptsig); + + for (let i = 0; i < pools.length; ++i) { + if (addresses.length) { + const poolAddresses: string[] = typeof pools[i].addresses === 'string' ? + JSON.parse(pools[i].addresses) : pools[i].addresses; + for (let y = 0; y < poolAddresses.length; y++) { + if (addresses.indexOf(poolAddresses[y]) !== -1) { + return pools[i]; + } + } + } + + const regexes: string[] = typeof pools[i].regexes === 'string' ? + JSON.parse(pools[i].regexes) : pools[i].regexes; + for (let y = 0; y < regexes.length; ++y) { + const regex = new RegExp(regexes[y], 'i'); + const match = asciiScriptSig.match(regex); + if (match !== null) { + return pools[i]; } } } @@ -134,68 +176,47 @@ class PoolsParser { } /** - * Delete indexed blocks for an updated mining pool - * - * @param pool + * re-index pool assignment for blocks previously associated with pool + * + * @param pool local id of existing pool to reindex */ - private async $deleteBlocksForPool(pool: PoolTag): Promise { - // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years - // Ignore early days of Bitcoin as there were no mining pool yet - const [oldestPoolBlock]: any[] = await DB.query(` - SELECT height + private async $reindexBlocksForPool(poolId: number): Promise { + let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52 + if (config.MEMPOOL.NETWORK === 'testnet') { + firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581 + } else if (config.MEMPOOL.NETWORK === 'signet') { + firstKnownBlockPool = 0; + } + + const [blocks]: any[] = await DB.query(` + SELECT height, hash, coinbase_raw, coinbase_addresses FROM blocks WHERE pool_id = ? - ORDER BY height - LIMIT 1`, - [pool.id] - ); + AND height >= ? + ORDER BY height DESC + `, [poolId, firstKnownBlockPool]); - let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52 - if (config.MEMPOOL.NETWORK === 'testnet') { - firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581 - } else if (config.MEMPOOL.NETWORK === 'signet') { - firstKnownBlockPool = 0; + let pools: PoolTag[] = []; + if (config.DATABASE.ENABLED === true) { + pools = await PoolsRepository.$getPools(); + } else { + pools = this.miningPools; } - const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : firstKnownBlockPool; - const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`); - this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`); - await DB.query(` - DELETE FROM blocks - WHERE pool_id = ? AND height >= ${oldestBlockHeight}`, - [unknownPool[0].id] - ); - logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`); - await DB.query(` - DELETE FROM blocks - WHERE pool_id = ?`, - [pool.id] - ); + let changed = 0; + for (const block of blocks) { + const addresses = JSON.parse(block.coinbase_addresses) || []; + const newPool = this.matchBlockMiner(block.coinbase_raw, addresses, pools); + if (newPool && newPool.id !== poolId) { + changed++; + await BlocksRepository.$savePool(block.hash, newPool.id); + } + } + + logger.info(`${changed} blocks assigned to a new pool`, logger.tags.mining); // Re-index hashrates and difficulty adjustments later mining.reindexHashrateRequested = true; - mining.reindexDifficultyAdjustmentRequested = true; - } - - private async $deleteUnknownBlocks(): Promise { - let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52 - if (config.MEMPOOL.NETWORK === 'testnet') { - firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581 - } else if (config.MEMPOOL.NETWORK === 'signet') { - firstKnownBlockPool = 0; - } - - const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`); - this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${firstKnownBlockPool} for re-indexing`); - await DB.query(` - DELETE FROM blocks - WHERE pool_id = ? AND height >= ${firstKnownBlockPool}`, - [unknownPool[0].id] - ); - - // Re-index hashrates and difficulty adjustments later - mining.reindexHashrateRequested = true; - mining.reindexDifficultyAdjustmentRequested = true; } } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index b2737b46d..90100a767 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -5,7 +5,7 @@ import logger from '../logger'; import { Common } from '../api/common'; import PoolsRepository from './PoolsRepository'; import HashratesRepository from './HashratesRepository'; -import { RowDataPacket, escape } from 'mysql2'; +import { RowDataPacket } from 'mysql2'; import BlocksSummariesRepository from './BlocksSummariesRepository'; import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository'; import bitcoinClient from '../api/bitcoin/bitcoin-client'; @@ -532,7 +532,7 @@ class BlocksRepository { return null; } - return await this.formatDbBlockIntoExtendedBlock(rows[0] as DatabaseBlock); + return await this.formatDbBlockIntoExtendedBlock(rows[0] as DatabaseBlock); } catch (e) { logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e)); throw e; @@ -1001,6 +1001,25 @@ class BlocksRepository { } } + /** + * Save pool + * + * @param id + * @param poolId + */ + public async $savePool(id: string, poolId: number): Promise { + try { + await DB.query(` + UPDATE blocks SET pool_id = ? + WHERE hash = ?`, + [poolId, id] + ); + } catch (e) { + logger.err(`Cannot update block pool. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + /** * Convert a mysql row block into a BlockExtended. Note that you * must provide the correct field into dbBlk object param