mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2025-01-18 21:32:55 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #5207 from mempool/mononaut/pool-reindexing

Browse Source 
Pool reindexing
This commit is contained in:
softsimon 2024-07-01 11:32:03 +09:00 committed by GitHub
commit 300bfd225b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 124 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/README.md
View File

@ -235,7 +235,7 @@ To manually update your mining pools, you can use the `--update-pools` command l
You can enable the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionally, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
When a `coinbase tag` or `coinbase address` change is detected, pool assignments for all relevant blocks (tagged to that pool or the `unknown` mining pool, starting from height 130635) are updated using the new criteria.
### Re-index tables

26
backend/src/api/blocks.ts
View File

@ -372,8 +372,7 @@ class Blocks {
}
}
const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig);
const addresses = txMinerInfo.vout.map((vout) => vout.scriptpubkey_address).filter((address) => address);
const addresses = txMinerInfo.vout.map((vout) => vout.scriptpubkey_address).filter(address => address) as string[];
let pools: PoolTag[] = [];
if (config.DATABASE.ENABLED === true) {
@ -382,26 +381,9 @@ class Blocks {
pools = poolsParser.miningPools;
}
for (let i = 0; i < pools.length; ++i) {
if (addresses.length) {
const poolAddresses: string[] = typeof pools[i].addresses === 'string' ?
JSON.parse(pools[i].addresses) : pools[i].addresses;
for (let y = 0; y < poolAddresses.length; y++) {
if (addresses.indexOf(poolAddresses[y]) !== -1) {
return pools[i];
}
}
}
const regexes: string[] = typeof pools[i].regexes === 'string' ?
JSON.parse(pools[i].regexes) : pools[i].regexes;
for (let y = 0; y < regexes.length; ++y) {
const regex = new RegExp(regexes[y], 'i');
const match = asciiScriptSig.match(regex);
if (match !== null) {
return pools[i];
}
}
const pool = poolsParser.matchBlockMiner(txMinerInfo.vin[0].scriptsig, addresses || [], pools);
if (pool) {
return pool;
}
if (config.DATABASE.ENABLED === true) {

145
backend/src/api/pools-parser.ts
View File

@ -5,6 +5,9 @@ import PoolsRepository from '../repositories/PoolsRepository';
import { PoolTag } from '../mempool.interfaces';
import diskCache from './disk-cache';
import mining from './mining/mining';
import transactionUtils from './transaction-utils';
import BlocksRepository from '../repositories/BlocksRepository';
import redisCache from './redis-cache';
class PoolsParser {
miningPools: any[] = [];
@ -37,15 +40,18 @@ class PoolsParser {
/**
* Populate our db with updated mining pool definition
* @param pools
* @param pools
*/
public async migratePoolsJson(): Promise<void> {
// We also need to wipe the backend cache to make sure we don't serve blocks with
// the wrong mining pool (usually happen with unknown blocks)
diskCache.setIgnoreBlocksCache();
redisCache.setIgnoreBlocksCache();
await this.$insertUnknownPool();
let reindexUnknown = false;
for (const pool of this.miningPools) {
if (!pool.id) {
logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`);
@ -57,22 +63,22 @@ class PoolsParser {
logger.err(`Mining pool ${pool.name} must have at least one of the fields 'addresses' or 'regexes'. Skipping.`);
continue;
}
pool.addresses = pool.addresses || [];
pool.regexes = pool.regexes || [];
if (pool.addresses.length === 0 && pool.regexes.length === 0) {
logger.err(`Mining pool ${pool.name} has no 'addresses' nor 'regexes' defined. Skipping.`);
continue;
}
if (pool.addresses.length === 0) {
logger.warn(`Mining pool ${pool.name} has no 'addresses' defined.`);
}
if (pool.regexes.length === 0) {
logger.warn(`Mining pool ${pool.name} has no 'regexes' defined.`);
}
}
const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false);
if (!poolDB) {
@ -80,7 +86,7 @@ class PoolsParser {
const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
logger.debug(`Inserting new mining pool ${pool.name}`);
await PoolsRepository.$insertNewMiningPool(pool, slug);
await this.$deleteUnknownBlocks();
reindexUnknown = true;
} else {
if (poolDB.name !== pool.name) {
// Pool has been renamed
@ -98,7 +104,45 @@ class PoolsParser {
// Pool addresses changed or coinbase tags changed
logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool.`);
await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
await this.$deleteBlocksForPool(poolDB);
reindexUnknown = true;
await this.$reindexBlocksForPool(poolDB.id);
}
}
}
if (reindexUnknown) {
logger.notice(`Updating addresses and/or coinbase tags for unknown mining pool.`);
let unknownPool;
if (config.DATABASE.ENABLED === true) {
unknownPool = await PoolsRepository.$getUnknownPool();
} else {
unknownPool = this.unknownPool;
}
await this.$reindexBlocksForPool(unknownPool.id);
}
}
public matchBlockMiner(scriptsig: string, addresses: string[], pools: PoolTag[]): PoolTag | undefined {
const asciiScriptSig = transactionUtils.hex2ascii(scriptsig);
for (let i = 0; i < pools.length; ++i) {
if (addresses.length) {
const poolAddresses: string[] = typeof pools[i].addresses === 'string' ?
JSON.parse(pools[i].addresses) : pools[i].addresses;
for (let y = 0; y < poolAddresses.length; y++) {
if (addresses.indexOf(poolAddresses[y]) !== -1) {
return pools[i];
}
}
}
const regexes: string[] = typeof pools[i].regexes === 'string' ?
JSON.parse(pools[i].regexes) : pools[i].regexes;
for (let y = 0; y < regexes.length; ++y) {
const regex = new RegExp(regexes[y], 'i');
const match = asciiScriptSig.match(regex);
if (match !== null) {
return pools[i];
}
}
}
@ -134,68 +178,47 @@ class PoolsParser {
}
/**
* Delete indexed blocks for an updated mining pool
*
* @param pool
* re-index pool assignment for blocks previously associated with pool
*
* @param pool local id of existing pool to reindex
*/
private async $deleteBlocksForPool(pool: PoolTag): Promise<void> {
// Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
// Ignore early days of Bitcoin as there were no mining pool yet
const [oldestPoolBlock]: any[] = await DB.query(`
SELECT height
private async $reindexBlocksForPool(poolId: number): Promise<void> {
let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
if (config.MEMPOOL.NETWORK === 'testnet') {
firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
} else if (config.MEMPOOL.NETWORK === 'signet') {
firstKnownBlockPool = 0;
}
const [blocks]: any[] = await DB.query(`
SELECT height, hash, coinbase_raw, coinbase_addresses
FROM blocks
WHERE pool_id = ?
ORDER BY height
LIMIT 1`,
[pool.id]
);
AND height >= ?
ORDER BY height DESC
`, [poolId, firstKnownBlockPool]);
let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
if (config.MEMPOOL.NETWORK === 'testnet') {
firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
} else if (config.MEMPOOL.NETWORK === 'signet') {
firstKnownBlockPool = 0;
let pools: PoolTag[] = [];
if (config.DATABASE.ENABLED === true) {
pools = await PoolsRepository.$getPools();
} else {
pools = this.miningPools;
}
const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : firstKnownBlockPool;
const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
await DB.query(`
DELETE FROM blocks
WHERE pool_id = ? AND height >= ${oldestBlockHeight}`,
[unknownPool[0].id]
);
logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`);
await DB.query(`
DELETE FROM blocks
WHERE pool_id = ?`,
[pool.id]
);
let changed = 0;
for (const block of blocks) {
const addresses = JSON.parse(block.coinbase_addresses) || [];
const newPool = this.matchBlockMiner(block.coinbase_raw, addresses, pools);
if (newPool && newPool.id !== poolId) {
changed++;
await BlocksRepository.$savePool(block.hash, newPool.id);
}
}
logger.info(`${changed} blocks assigned to a new pool`, logger.tags.mining);
// Re-index hashrates and difficulty adjustments later
mining.reindexHashrateRequested = true;
mining.reindexDifficultyAdjustmentRequested = true;
}
private async $deleteUnknownBlocks(): Promise<void> {
let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
if (config.MEMPOOL.NETWORK === 'testnet') {
firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
} else if (config.MEMPOOL.NETWORK === 'signet') {
firstKnownBlockPool = 0;
}
const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${firstKnownBlockPool} for re-indexing`);
await DB.query(`
DELETE FROM blocks
WHERE pool_id = ? AND height >= ${firstKnownBlockPool}`,
[unknownPool[0].id]
);
// Re-index hashrates and difficulty adjustments later
mining.reindexHashrateRequested = true;
mining.reindexDifficultyAdjustmentRequested = true;
}
}

20
backend/src/api/redis-cache.ts
View File

@ -27,6 +27,7 @@ class RedisCache {
private rbfCacheQueue: { type: string, txid: string, value: any }[] = [];
private rbfRemoveQueue: { type: string, txid: string }[] = [];
private txFlushLimit: number = 10000;
private ignoreBlocksCache = false;
constructor() {
if (config.REDIS.ENABLED) {
@ -341,9 +342,7 @@ class RedisCache {
return;
}
logger.info('Restoring mempool and blocks data from Redis cache');
// Load block data
const loadedBlocks = await this.$getBlocks();
const loadedBlockSummaries = await this.$getBlockSummaries();
// Load mempool
const loadedMempool = await this.$getMempool();
this.inflateLoadedTxs(loadedMempool);
@ -352,9 +351,14 @@ class RedisCache {
const rbfTrees = await this.$getRbfEntries('tree');
const rbfExpirations = await this.$getRbfEntries('exp');
// Set loaded data
blocks.setBlocks(loadedBlocks || []);
blocks.setBlockSummaries(loadedBlockSummaries || []);
// Load & set block data
if (!this.ignoreBlocksCache) {
const loadedBlocks = await this.$getBlocks();
const loadedBlockSummaries = await this.$getBlockSummaries();
blocks.setBlocks(loadedBlocks || []);
blocks.setBlockSummaries(loadedBlockSummaries || []);
}
// Set other data
await memPool.$setMempool(loadedMempool);
await rbfCache.load({
txs: rbfTxs,
@ -411,6 +415,10 @@ class RedisCache {
}
return result;
}
public setIgnoreBlocksCache(): void {
this.ignoreBlocksCache = true;
}
}
export default new RedisCache();

23
backend/src/repositories/BlocksRepository.ts
View File

@ -5,7 +5,7 @@ import logger from '../logger';
import { Common } from '../api/common';
import PoolsRepository from './PoolsRepository';
import HashratesRepository from './HashratesRepository';
import { RowDataPacket, escape } from 'mysql2';
import { RowDataPacket } from 'mysql2';
import BlocksSummariesRepository from './BlocksSummariesRepository';
import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
import bitcoinClient from '../api/bitcoin/bitcoin-client';
@ -532,7 +532,7 @@ class BlocksRepository {
return null;
}
return await this.formatDbBlockIntoExtendedBlock(rows[0] as DatabaseBlock);
return await this.formatDbBlockIntoExtendedBlock(rows[0] as DatabaseBlock);
} catch (e) {
logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
@ -1001,6 +1001,25 @@ class BlocksRepository {
}
}
/**
* Save pool
*
* @param id
* @param poolId
*/
public async $savePool(id: string, poolId: number): Promise<void> {
try {
await DB.query(`
UPDATE blocks SET pool_id = ?
WHERE hash = ?`,
[poolId, id]
);
} catch (e) {
logger.err(`Cannot update block pool. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Convert a mysql row block into a BlockExtended. Note that you
* must provide the correct field into dbBlk object param