From 030020ea9e930d1f9a21219e11f24fb86b1ec80f Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 9 Jul 2022 18:25:16 +0200 Subject: [PATCH 1/5] Run the mining indexer every hour to index missing/updated data --- backend/src/indexer.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 8e4e7e87f..e34c43e8d 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -35,6 +35,8 @@ class Indexer { this.runIndexer = false; this.indexerRunning = true; + logger.debug(`Running mining indexer`); + try { const chainValid = await blocks.$generateBlockDatabase(); if (chainValid === false) { @@ -54,9 +56,15 @@ class Indexer { this.indexerRunning = false; logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e)); setTimeout(() => this.reindex(), 10000); + this.indexerRunning = false; + return; } this.indexerRunning = false; + + const runEvery = 1000 * 3600; // 1 hour + logger.debug(`Indexing completed. Next run planned at ${new Date(new Date().getTime() + runEvery).toUTCString()}`); + setTimeout(() => this.reindex(), runEvery); } async $resetHashratesIndexingState() { From e303a4c37440883762d5621de1800300baf65dd5 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 8 Jul 2022 11:15:48 +0200 Subject: [PATCH 2/5] Removed hardcoded genesis timestamp Fix duplicated genesis hashrate attempt Add log during difficulty adjustment indexing --- backend/src/api/mining.ts | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index d2504274c..04da1049d 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -189,10 +189,12 @@ class Mining { } try { + const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); + const genesisTimestamp = genesisBlock.time * 1000; + const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); const hashrates: any[] = []; - const genesisTimestamp = 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f - + const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7)); const lastMondayMidnight = this.getDateMidnight(lastMonday); let toTimestamp = lastMondayMidnight.getTime(); @@ -297,8 +299,9 @@ class Mining { } try { + const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); + const genesisTimestamp = genesisBlock.time * 1000; const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); - const genesisTimestamp = (config.MEMPOOL.NETWORK === 'signet') ? 1598918400000 : 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f const lastMidnight = this.getDateMidnight(new Date()); let toTimestamp = Math.round(lastMidnight.getTime()); const hashrates: any[] = []; @@ -368,8 +371,8 @@ class Mining { ++totalIndexed; } - // Add genesis block manually on mainnet and testnet - if ('signet' !== config.MEMPOOL.NETWORK && toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) { + // Add genesis block manually + if (!indexedTimestamp.includes(genesisTimestamp / 1000)) { hashrates.push({ hashrateTimestamp: genesisTimestamp / 1000, avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1), @@ -410,14 +413,18 @@ class Mining { let totalIndexed = 0; if (indexedHeights[0] !== true) { + const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); await DifficultyAdjustmentsRepository.$saveAdjustments({ - time: (config.MEMPOOL.NETWORK === 'signet') ? 1598918400 : 1231006505, + time: genesisBlock.time, height: 0, - difficulty: (config.MEMPOOL.NETWORK === 'signet') ? 0.001126515290698186 : 1.0, + difficulty: genesisBlock.difficulty, adjustment: 0.0, }); } + let totalBlockChecked = 0; + let timer = new Date().getTime() / 1000; + for (const block of blocks) { if (block.difficulty !== currentDifficulty) { if (block.height === 0 || indexedHeights[block.height] === true) { // Already indexed @@ -438,6 +445,14 @@ class Mining { totalIndexed++; currentDifficulty = block.difficulty; } + + totalBlockChecked++; + const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); + if (elapsedSeconds > 5) { + const progress = Math.round(totalBlockChecked / blocks.length * 100); + logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`); + timer = new Date().getTime() / 1000; + } } if (totalIndexed > 0) { From d8a90cce47e667fdc5fc609466ec1842b9ebdabc Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 9 Jul 2022 10:26:31 +0200 Subject: [PATCH 3/5] Use raw db hashrate instead of avg for indexing --- backend/src/api/mining.ts | 35 +++++++------------ .../src/repositories/HashratesRepository.ts | 26 ++++++++++++++ 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 04da1049d..de59e9b58 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -173,19 +173,14 @@ class Mining { */ public async $generatePoolHashrateHistory(): Promise { const now = new Date(); + const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing'); - try { - const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing'); - - // Run only if: - // * lastestRunDate is set to 0 (node backend restart, reorg) - // * we started a new week (around Monday midnight) - const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate(); - if (!runIndexing) { - return; - } - } catch (e) { - throw e; + // Run only if: + // * lastestRunDate is set to 0 (node backend restart, reorg) + // * we started a new week (around Monday midnight) + const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate(); + if (!runIndexing) { + return; } try { @@ -287,21 +282,17 @@ class Mining { * [INDEXING] Generate daily hashrate data */ public async $generateNetworkHashrateHistory(): Promise { - try { - // We only run this once a day around midnight - const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing'); - const now = new Date().getUTCDate(); - if (now === latestRunDate) { - return; - } - } catch (e) { - throw e; + // We only run this once a day around midnight + const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing'); + const now = new Date().getUTCDate(); + if (now === latestRunDate) { + return; } try { const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); const genesisTimestamp = genesisBlock.time * 1000; - const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); + const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); const lastMidnight = this.getDateMidnight(new Date()); let toTimestamp = Math.round(lastMidnight.getTime()); const hashrates: any[] = []; diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 5e6048abc..ae1eabb30 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -30,6 +30,32 @@ class HashratesRepository { } } + public async $getRawNetworkDailyHashrate(interval: string | null): Promise { + interval = Common.getSqlInterval(interval); + + let query = `SELECT + UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, + avg_hashrate as avgHashrate + FROM hashrates`; + + if (interval) { + query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() + AND hashrates.type = 'daily'`; + } else { + query += ` WHERE hashrates.type = 'daily'`; + } + + query += ` ORDER by hashrate_timestamp`; + + try { + const [rows]: any[] = await DB.query(query); + return rows; + } catch (e) { + logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $getNetworkDailyHashrate(interval: string | null): Promise { interval = Common.getSqlInterval(interval); From 067ee168dd7d9fe4500fb731f908733d88e69117 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 9 Jul 2022 12:05:36 +0200 Subject: [PATCH 4/5] Use oldest consecutive block timestamp as hashrate indexing limit --- backend/src/api/mining.ts | 28 ++++++-------------- backend/src/repositories/BlocksRepository.ts | 18 +++++++++++++ 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index de59e9b58..7f142922b 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -184,6 +184,8 @@ class Mining { } try { + const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlockTimestamp()); + const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); const genesisTimestamp = genesisBlock.time * 1000; @@ -204,7 +206,7 @@ class Mining { logger.debug(`Indexing weekly mining pool hashrate`); loadingIndicators.setProgress('weekly-hashrate-indexing', 0); - while (toTimestamp > genesisTimestamp) { + while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) { const fromTimestamp = toTimestamp - 604800000; // Skip already indexed weeks @@ -214,14 +216,6 @@ class Mining { continue; } - // Check if we have blocks for the previous week (which mean that the week - // we are currently indexing has complete data) - const blockStatsPreviousWeek: any = await BlocksRepository.$blockCountBetweenTimestamp( - null, (fromTimestamp - 604800000) / 1000, (toTimestamp - 604800000) / 1000); - if (blockStatsPreviousWeek.blockCount === 0) { // We are done indexing - break; - } - const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp( null, fromTimestamp / 1000, toTimestamp / 1000); const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, @@ -289,6 +283,8 @@ class Mining { return; } + const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlockTimestamp()); + try { const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); const genesisTimestamp = genesisBlock.time * 1000; @@ -307,7 +303,7 @@ class Mining { logger.debug(`Indexing daily network hashrate`); loadingIndicators.setProgress('daily-hashrate-indexing', 0); - while (toTimestamp > genesisTimestamp) { + while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) { const fromTimestamp = toTimestamp - 86400000; // Skip already indexed days @@ -317,17 +313,9 @@ class Mining { continue; } - // Check if we have blocks for the previous day (which mean that the day - // we are currently indexing has complete data) - const blockStatsPreviousDay: any = await BlocksRepository.$blockCountBetweenTimestamp( - null, (fromTimestamp - 86400000) / 1000, (toTimestamp - 86400000) / 1000); - if (blockStatsPreviousDay.blockCount === 0 && config.MEMPOOL.NETWORK === 'mainnet') { // We are done indexing - break; - } - const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp( null, fromTimestamp / 1000, toTimestamp / 1000); - const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, + const lastBlockHashrate = blockStats.blockCount === 0 ? 0 : await bitcoinClient.getNetworkHashPs(blockStats.blockCount, blockStats.lastBlockHeight); hashrates.push({ @@ -363,7 +351,7 @@ class Mining { } // Add genesis block manually - if (!indexedTimestamp.includes(genesisTimestamp / 1000)) { + if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && !indexedTimestamp.includes(genesisTimestamp / 1000)) { hashrates.push({ hashrateTimestamp: genesisTimestamp / 1000, avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1), diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index c6b14ff51..c3f824d61 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -610,6 +610,24 @@ class BlocksRepository { throw e; } } + + /** + * Return the oldest block timestamp from a consecutive chain of block from the most recent one + */ + public async $getOldestConsecutiveBlockTimestamp(): Promise { + try { + const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height DESC`); + for (let i = 0; i < rows.length - 1; ++i) { + if (rows[i].height - rows[i + 1].height > 1) { + return rows[i].timestamp; + } + } + return rows[rows.length - 1].timestamp; + } catch (e) { + logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } } export default new BlocksRepository(); From 07cb4a49bc3da883d1b22116536196200d602e9c Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 9 Jul 2022 22:34:18 +0200 Subject: [PATCH 5/5] Index weekly mining pool hashrate only if there are blocks mined --- backend/src/api/mining.ts | 36 ++++++++++--------- .../src/repositories/HashratesRepository.ts | 1 - 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 7f142922b..207625be0 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -223,25 +223,27 @@ class Mining { let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp / 1000, toTimestamp / 1000); const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0); - pools = pools.map((pool: any) => { - pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate; - pool.share = (pool.blockCount / totalBlocks); - return pool; - }); - - for (const pool of pools) { - hashrates.push({ - hashrateTimestamp: toTimestamp / 1000, - avgHashrate: pool['hashrate'], - poolId: pool.poolId, - share: pool['share'], - type: 'weekly', + if (totalBlocks > 0) { + pools = pools.map((pool: any) => { + pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate; + pool.share = (pool.blockCount / totalBlocks); + return pool; }); - } - newlyIndexed += hashrates.length; - await HashratesRepository.$saveHashrates(hashrates); - hashrates.length = 0; + for (const pool of pools) { + hashrates.push({ + hashrateTimestamp: toTimestamp / 1000, + avgHashrate: pool['hashrate'] , + poolId: pool.poolId, + share: pool['share'], + type: 'weekly', + }); + } + + newlyIndexed += hashrates.length; + await HashratesRepository.$saveHashrates(hashrates); + hashrates.length = 0; + } const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); if (elapsedSeconds > 1) { diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index ae1eabb30..e5a193477 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -1,6 +1,5 @@ import { escape } from 'mysql2'; import { Common } from '../api/common'; -import config from '../config'; import DB from '../database'; import logger from '../logger'; import PoolsRepository from './PoolsRepository';