Merge branch 'master' into nymkappa/feature/merge-blocks

backend/src/api/blocks.ts

@@ -15,6 +15,7 @@ import BitcoinApi from './bitcoin/bitcoin-api';
 import { prepareBlock } from '../utils/blocks-utils';
 import BlocksRepository from '../repositories/BlocksRepository';
 import HashratesRepository from '../repositories/HashratesRepository';
+import indexer from '../indexer';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -23,9 +24,6 @@ class Blocks {
   private lastDifficultyAdjustmentTime = 0;
   private previousDifficultyRetarget = 0;
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
-  private blockIndexingStarted = false;
-  public blockIndexingCompleted = false;
-  public reindexFlag = false;
 
   constructor() { }
 
@@ -197,24 +195,15 @@ class Blocks {
    * [INDEXING] Index all blocks metadata for the mining dashboard
    */
   public async $generateBlockDatabase() {
-    if (this.blockIndexingStarted && !this.reindexFlag) {
-      return;
-    }
-
-    this.reindexFlag = false;
-
     const blockchainInfo = await bitcoinClient.getBlockchainInfo();
     if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync
       return;
     }
 
-    this.blockIndexingStarted = true;
-    this.blockIndexingCompleted = false;
-
     try {
       let currentBlockHeight = blockchainInfo.blocks;
 
-      let indexingBlockAmount = config.MEMPOOL.INDEXING_BLOCKS_AMOUNT;
+      let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks);
       if (indexingBlockAmount <= -1) {
         indexingBlockAmount = currentBlockHeight + 1;
       }
@@ -275,14 +264,14 @@ class Blocks {
       loadingIndicators.setProgress('block-indexing', 100);
     } catch (e) {
       logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));
-      this.blockIndexingStarted = false;
       loadingIndicators.setProgress('block-indexing', 100);
       return;
     }
 
     const chainValid = await BlocksRepository.$validateChain();
-    this.reindexFlag = !chainValid;
+    if (!chainValid) {
-    this.blockIndexingCompleted = chainValid;
+      indexer.reindex();
+    }
   }
 
   public async $updateBlocks() {
@@ -299,6 +288,8 @@ class Blocks {
       logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.MEMPOOL.INITIAL_BLOCKS_AMOUNT} recent blocks`);
       this.currentBlockHeight = blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT;
       fastForwarded = true;
+      logger.info(`Re-indexing skipped blocks and corresponding hashrates data`);
+      indexer.reindex(); // Make sure to index the skipped blocks #1619
     }
 
     if (!this.lastDifficultyAdjustmentTime) {

backend/src/api/mining.ts

@@ -4,15 +4,11 @@ import PoolsRepository from '../repositories/PoolsRepository';
 import HashratesRepository from '../repositories/HashratesRepository';
 import bitcoinClient from './bitcoin/bitcoin-client';
 import logger from '../logger';
-import blocks from './blocks';
 import { Common } from './common';
 import loadingIndicators from './loading-indicators';
 import { escape } from 'mysql2';
 
 class Mining {
-  hashrateIndexingStarted = false;
-  weeklyHashrateIndexingStarted = false;
-
   constructor() {
   }
 
@@ -153,14 +149,9 @@ class Mining {
    * [INDEXING] Generate weekly mining pool hashrate history
    */
   public async $generatePoolHashrateHistory(): Promise<void> {
-    if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted || this.weeklyHashrateIndexingStarted) {
-      return;
-    }
-
     const now = new Date();
 
     try {
-      this.weeklyHashrateIndexingStarted = true;
       const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
 
       // Run only if:
@@ -168,11 +159,9 @@ class Mining {
       // * we started a new week (around Monday midnight)
       const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
       if (!runIndexing) {
-        this.weeklyHashrateIndexingStarted = false;
         return;
       }
     } catch (e) {
-      this.weeklyHashrateIndexingStarted = false;
       throw e;
     }
 
@@ -192,6 +181,7 @@ class Mining {
       const startedAt = new Date().getTime() / 1000;
       let timer = new Date().getTime() / 1000;
 
+      logger.debug(`Indexing weekly mining pool hashrate`);
       loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
 
       while (toTimestamp > genesisTimestamp) {
@@ -256,7 +246,6 @@ class Mining {
         ++indexedThisRun;
         ++totalIndexed;
       }
-      this.weeklyHashrateIndexingStarted = false;
       await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
       if (newlyIndexed > 0) {
         logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`);
@@ -264,7 +253,6 @@ class Mining {
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
-      this.weeklyHashrateIndexingStarted = false;
       throw e;
     }
   }
@@ -273,22 +261,14 @@ class Mining {
    * [INDEXING] Generate daily hashrate data
    */
   public async $generateNetworkHashrateHistory(): Promise<void> {
-    if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) {
-      return;
-    }
-
     try {
-      this.hashrateIndexingStarted = true;
-
       // We only run this once a day around midnight
       const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
       const now = new Date().getUTCDate();
       if (now === latestRunDate) {
-        this.hashrateIndexingStarted = false;
         return;
       }
     } catch (e) {
-      this.hashrateIndexingStarted = false;
       throw e;
     }
 
@@ -306,6 +286,7 @@ class Mining {
       const startedAt = new Date().getTime() / 1000;
       let timer = new Date().getTime() / 1000;
 
+      logger.debug(`Indexing daily network hashrate`);
       loadingIndicators.setProgress('daily-hashrate-indexing', 0);
 
       while (toTimestamp > genesisTimestamp) {
@@ -377,14 +358,12 @@ class Mining {
       await HashratesRepository.$saveHashrates(hashrates);
 
       await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
-      this.hashrateIndexingStarted = false;
       if (newlyIndexed > 0) {
         logger.info(`Indexed ${newlyIndexed} day of network hashrate`);
       }
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
-      this.hashrateIndexingStarted = false;
       throw e;
     }
   }

backend/src/index.ts

@@ -28,6 +28,7 @@ import { Common } from './api/common';
 import mining from './api/mining';
 import HashratesRepository from './repositories/HashratesRepository';
 import poolsUpdater from './tasks/pools-updater';
+import indexer from './indexer';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -98,7 +99,7 @@ class Server {
         }
         await databaseMigration.$initializeOrMigrateDatabase();
         if (Common.indexingEnabled()) {
-          await this.$resetHashratesIndexingState();
+          await indexer.$resetHashratesIndexingState();
         }
       } catch (e) {
         throw new Error(e instanceof Error ? e.message : 'Error');
@@ -153,7 +154,7 @@ class Server {
       await poolsUpdater.updatePoolsJson();
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
-      this.$runIndexingWhenReady();
+      indexer.$run();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;
@@ -172,29 +173,6 @@ class Server {
     }
   }
 
-  async $resetHashratesIndexingState() {
-    try {
-      await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
-      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
-    } catch (e) {
-      logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  async $runIndexingWhenReady() {
-    if (!Common.indexingEnabled() || mempool.hasPriority()) {
-      return;
-    }
-
-    try {
-      await blocks.$generateBlockDatabase();
-      await mining.$generateNetworkHashrateHistory();
-      await mining.$generatePoolHashrateHistory();
-    } catch (e) {
-      logger.err(`Indexing failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e));
-    }
-  }
-
   setUpWebsocketHandling() {
     if (this.wss) {
       websocketHandler.setWebsocketServer(this.wss);

backend/src/indexer.ts

@@ -0,0 +1,52 @@
+import { Common } from './api/common';
+import blocks from './api/blocks';
+import mempool from './api/mempool';
+import mining from './api/mining';
+import logger from './logger';
+import HashratesRepository from './repositories/HashratesRepository';
+
+class Indexer {
+  runIndexer = true;
+  indexerRunning = false;
+
+  constructor() {
+  }
+
+  public reindex() {
+    this.runIndexer = true;
+  }
+
+  public async $run() {
+    if (!Common.indexingEnabled() || this.runIndexer === false ||
+      this.indexerRunning === true || mempool.hasPriority()
+    ) {
+      return;
+    }
+
+    this.runIndexer = false;
+    this.indexerRunning = true;
+
+    try {
+      await blocks.$generateBlockDatabase();
+      await this.$resetHashratesIndexingState();
+      await mining.$generateNetworkHashrateHistory();
+      await mining.$generatePoolHashrateHistory();
+    } catch (e) {
+      this.reindex();
+      logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+    this.indexerRunning = false;
+  }
+
+  async $resetHashratesIndexingState() {
+    try {
+      await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
+      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
+    } catch (e) {
+      logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+}
+
+export default new Indexer();