diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 91db74faa..b9b7b3aff 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -15,6 +15,7 @@ import BitcoinApi from './bitcoin/bitcoin-api'; import { prepareBlock } from '../utils/blocks-utils'; import BlocksRepository from '../repositories/BlocksRepository'; import HashratesRepository from '../repositories/HashratesRepository'; +import indexer from '../indexer'; class Blocks { private blocks: BlockExtended[] = []; @@ -23,9 +24,6 @@ class Blocks { private lastDifficultyAdjustmentTime = 0; private previousDifficultyRetarget = 0; private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; - private blockIndexingStarted = false; - public blockIndexingCompleted = false; - public reindexFlag = false; constructor() { } @@ -197,24 +195,15 @@ class Blocks { * [INDEXING] Index all blocks metadata for the mining dashboard */ public async $generateBlockDatabase() { - if (this.blockIndexingStarted && !this.reindexFlag) { - return; - } - - this.reindexFlag = false; - const blockchainInfo = await bitcoinClient.getBlockchainInfo(); if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync return; } - this.blockIndexingStarted = true; - this.blockIndexingCompleted = false; - try { let currentBlockHeight = blockchainInfo.blocks; - let indexingBlockAmount = config.MEMPOOL.INDEXING_BLOCKS_AMOUNT; + let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks); if (indexingBlockAmount <= -1) { indexingBlockAmount = currentBlockHeight + 1; } @@ -275,14 +264,14 @@ class Blocks { loadingIndicators.setProgress('block-indexing', 100); } catch (e) { logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e)); - this.blockIndexingStarted = false; loadingIndicators.setProgress('block-indexing', 100); return; } const chainValid = await BlocksRepository.$validateChain(); - this.reindexFlag = !chainValid; - this.blockIndexingCompleted = chainValid; + if (!chainValid) { + indexer.reindex(); + } } public async $updateBlocks() { @@ -299,6 +288,8 @@ class Blocks { logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.MEMPOOL.INITIAL_BLOCKS_AMOUNT} recent blocks`); this.currentBlockHeight = blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT; fastForwarded = true; + logger.info(`Re-indexing skipped blocks and corresponding hashrates data`); + indexer.reindex(); // Make sure to index the skipped blocks #1619 } if (!this.lastDifficultyAdjustmentTime) { diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 81821b6f7..8d11abfa7 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -4,15 +4,11 @@ import PoolsRepository from '../repositories/PoolsRepository'; import HashratesRepository from '../repositories/HashratesRepository'; import bitcoinClient from './bitcoin/bitcoin-client'; import logger from '../logger'; -import blocks from './blocks'; import { Common } from './common'; import loadingIndicators from './loading-indicators'; import { escape } from 'mysql2'; class Mining { - hashrateIndexingStarted = false; - weeklyHashrateIndexingStarted = false; - constructor() { } @@ -153,14 +149,9 @@ class Mining { * [INDEXING] Generate weekly mining pool hashrate history */ public async $generatePoolHashrateHistory(): Promise { - if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted || this.weeklyHashrateIndexingStarted) { - return; - } - const now = new Date(); try { - this.weeklyHashrateIndexingStarted = true; const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing'); // Run only if: @@ -168,11 +159,9 @@ class Mining { // * we started a new week (around Monday midnight) const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate(); if (!runIndexing) { - this.weeklyHashrateIndexingStarted = false; return; } } catch (e) { - this.weeklyHashrateIndexingStarted = false; throw e; } @@ -192,6 +181,7 @@ class Mining { const startedAt = new Date().getTime() / 1000; let timer = new Date().getTime() / 1000; + logger.debug(`Indexing weekly mining pool hashrate`); loadingIndicators.setProgress('weekly-hashrate-indexing', 0); while (toTimestamp > genesisTimestamp) { @@ -256,7 +246,6 @@ class Mining { ++indexedThisRun; ++totalIndexed; } - this.weeklyHashrateIndexingStarted = false; await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate()); if (newlyIndexed > 0) { logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`); @@ -264,7 +253,6 @@ class Mining { loadingIndicators.setProgress('weekly-hashrate-indexing', 100); } catch (e) { loadingIndicators.setProgress('weekly-hashrate-indexing', 100); - this.weeklyHashrateIndexingStarted = false; throw e; } } @@ -273,22 +261,14 @@ class Mining { * [INDEXING] Generate daily hashrate data */ public async $generateNetworkHashrateHistory(): Promise { - if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) { - return; - } - try { - this.hashrateIndexingStarted = true; - // We only run this once a day around midnight const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing'); const now = new Date().getUTCDate(); if (now === latestRunDate) { - this.hashrateIndexingStarted = false; return; } } catch (e) { - this.hashrateIndexingStarted = false; throw e; } @@ -306,6 +286,7 @@ class Mining { const startedAt = new Date().getTime() / 1000; let timer = new Date().getTime() / 1000; + logger.debug(`Indexing daily network hashrate`); loadingIndicators.setProgress('daily-hashrate-indexing', 0); while (toTimestamp > genesisTimestamp) { @@ -377,14 +358,12 @@ class Mining { await HashratesRepository.$saveHashrates(hashrates); await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate()); - this.hashrateIndexingStarted = false; if (newlyIndexed > 0) { logger.info(`Indexed ${newlyIndexed} day of network hashrate`); } loadingIndicators.setProgress('daily-hashrate-indexing', 100); } catch (e) { loadingIndicators.setProgress('daily-hashrate-indexing', 100); - this.hashrateIndexingStarted = false; throw e; } } diff --git a/backend/src/index.ts b/backend/src/index.ts index 09d8f7669..30c5ecf37 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -28,6 +28,7 @@ import { Common } from './api/common'; import mining from './api/mining'; import HashratesRepository from './repositories/HashratesRepository'; import poolsUpdater from './tasks/pools-updater'; +import indexer from './indexer'; class Server { private wss: WebSocket.Server | undefined; @@ -98,7 +99,7 @@ class Server { } await databaseMigration.$initializeOrMigrateDatabase(); if (Common.indexingEnabled()) { - await this.$resetHashratesIndexingState(); + await indexer.$resetHashratesIndexingState(); } } catch (e) { throw new Error(e instanceof Error ? e.message : 'Error'); @@ -153,7 +154,7 @@ class Server { await poolsUpdater.updatePoolsJson(); await blocks.$updateBlocks(); await memPool.$updateMempool(); - this.$runIndexingWhenReady(); + indexer.$run(); setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); this.currentBackendRetryInterval = 5; @@ -172,29 +173,6 @@ class Server { } } - async $resetHashratesIndexingState() { - try { - await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0); - await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0); - } catch (e) { - logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e)); - } - } - - async $runIndexingWhenReady() { - if (!Common.indexingEnabled() || mempool.hasPriority()) { - return; - } - - try { - await blocks.$generateBlockDatabase(); - await mining.$generateNetworkHashrateHistory(); - await mining.$generatePoolHashrateHistory(); - } catch (e) { - logger.err(`Indexing failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e)); - } - } - setUpWebsocketHandling() { if (this.wss) { websocketHandler.setWebsocketServer(this.wss); diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts new file mode 100644 index 000000000..7ddc2a47f --- /dev/null +++ b/backend/src/indexer.ts @@ -0,0 +1,52 @@ +import { Common } from './api/common'; +import blocks from './api/blocks'; +import mempool from './api/mempool'; +import mining from './api/mining'; +import logger from './logger'; +import HashratesRepository from './repositories/HashratesRepository'; + +class Indexer { + runIndexer = true; + indexerRunning = false; + + constructor() { + } + + public reindex() { + this.runIndexer = true; + } + + public async $run() { + if (!Common.indexingEnabled() || this.runIndexer === false || + this.indexerRunning === true || mempool.hasPriority() + ) { + return; + } + + this.runIndexer = false; + this.indexerRunning = true; + + try { + await blocks.$generateBlockDatabase(); + await this.$resetHashratesIndexingState(); + await mining.$generateNetworkHashrateHistory(); + await mining.$generatePoolHashrateHistory(); + } catch (e) { + this.reindex(); + logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e)); + } + + this.indexerRunning = false; + } + + async $resetHashratesIndexingState() { + try { + await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0); + await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0); + } catch (e) { + logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e)); + } + } +} + +export default new Indexer();