Merge branch 'master' into simon/shorten-address-prefix-result

backend/src/api/bitcoin/bitcoin-api.ts

@@ -84,19 +84,19 @@ class BitcoinApi implements AbstractBitcoinApi {
   }
 
   $getAddressPrefix(prefix: string): string[] {
-    const found: string[] = [];
+    const found: { [address: string]: string } = {};
     const mp = mempool.getMempool();
     for (const tx in mp) {
       for (const vout of mp[tx].vout) {
         if (vout.scriptpubkey_address.indexOf(prefix) === 0) {
-          found.push(vout.scriptpubkey_address);
-          if (found.length >= 10) {
-            return found;
+          found[vout.scriptpubkey_address] = '';
+          if (Object.keys(found).length >= 10) {
+            return Object.keys(found);
           }
         }
       }
     }
-    return found;
+    return Object.keys(found);
   }
 
   $sendRawTransaction(rawTransaction: string): Promise<string> {

backend/src/api/blocks.ts

@@ -218,31 +218,28 @@ class Blocks {
           if (blockHeight < lastBlockToIndex) {
             break;
           }
-          try {
-            ++indexedThisRun;
-            if (++totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) {
-              const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
-              const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
-              const progress = Math.round(totaIndexed / indexingBlockAmount * 100);
-              const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds);
-              logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${elapsedSeconds} seconds | left: ~${timeLeft} seconds`);
-            }
-            const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
-            const block = await bitcoinApi.$getBlock(blockHash);
-            const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
-            const blockExtended = await this.$getBlockExtended(block, transactions);
-            await blocksRepository.$saveBlockInDatabase(blockExtended);
-          } catch (e) {
-            logger.err(`Something went wrong while indexing blocks.` + e);
+          ++indexedThisRun;
+          if (++totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) {
+            const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+            const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+            const progress = Math.round(totaIndexed / indexingBlockAmount * 100);
+            const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds);
+            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${elapsedSeconds} seconds | left: ~${timeLeft} seconds`);
           }
+          const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
+          const block = await bitcoinApi.$getBlock(blockHash);
+          const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
+          const blockExtended = await this.$getBlockExtended(block, transactions);
+          await blocksRepository.$saveBlockInDatabase(blockExtended);
         }
 
         currentBlockHeight -= chunkSize;
       }
       logger.info('Block indexing completed');
     } catch (e) {
-      logger.err('An error occured in $generateBlockDatabase(). Skipping block indexing. ' + e);
-      console.log(e);
+      logger.err('An error occured in $generateBlockDatabase(). Trying again later. ' + e);
+      this.blockIndexingStarted = false;
+      return;
     }
 
     this.blockIndexingCompleted = true;

backend/src/api/mining.ts

@@ -90,7 +90,7 @@ class Mining {
   /**
    * Return the historical hashrates and oldest indexed block timestamp for one or all pools
    */
-   public async $getPoolsHistoricalHashrates(interval: string | null, poolId: number): Promise<object> {
+  public async $getPoolsHistoricalHashrates(interval: string | null, poolId: number): Promise<object> {
     return await HashratesRepository.$getPoolsWeeklyHashrate(interval);
   }
 
@@ -108,106 +108,112 @@ class Mining {
     if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) {
       return;
     }
-    this.hashrateIndexingStarted = true;
 
-    logger.info(`Indexing hashrates`);
+    try {
+      this.hashrateIndexingStarted = true;
 
-    const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144;
-    const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
-    let startedAt = new Date().getTime() / 1000;
-    const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
-    const lastMidnight = new Date();
-    lastMidnight.setUTCHours(0); lastMidnight.setUTCMinutes(0); lastMidnight.setUTCSeconds(0); lastMidnight.setUTCMilliseconds(0);
-    let toTimestamp = Math.round(lastMidnight.getTime() / 1000);
-    let indexedThisRun = 0;
-    let totalIndexed = 0;
+      logger.info(`Indexing hashrates`);
 
-    const hashrates: any[] = [];
+      const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144;
+      const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
+      let startedAt = new Date().getTime() / 1000;
+      const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
+      const lastMidnight = new Date();
+      lastMidnight.setUTCHours(0); lastMidnight.setUTCMinutes(0); lastMidnight.setUTCSeconds(0); lastMidnight.setUTCMilliseconds(0);
+      let toTimestamp = Math.round(lastMidnight.getTime() / 1000);
+      let indexedThisRun = 0;
+      let totalIndexed = 0;
 
-    while (toTimestamp > genesisTimestamp) {
-      const fromTimestamp = toTimestamp - 86400;
-      if (indexedTimestamp.includes(fromTimestamp)) {
-        toTimestamp -= 86400;
-        ++totalIndexed;
-        continue;
-      }
+      const hashrates: any[] = [];
 
-      const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
-        null, fromTimestamp, toTimestamp);
-      if (blockStats.blockCount === 0) { // We are done indexing, no blocks left
-        break;
-      }
-
-      let lastBlockHashrate = 0;
-      lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
-        blockStats.lastBlockHeight);
-
-      if (totalIndexed % 7 === 0 && !indexedTimestamp.includes(fromTimestamp + 1)) { // Save weekly pools hashrate
-        logger.debug("Indexing weekly hashrates for mining pools");
-        let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp - 604800, fromTimestamp);
-        const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
-        pools = pools.map((pool: any) => {
-          pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
-          pool.share = (pool.blockCount / totalBlocks);
-          return pool;
-        });
-  
-        for (const pool of pools) {
-          hashrates.push({
-            hashrateTimestamp: fromTimestamp + 1,
-            avgHashrate: pool['hashrate'],
-            poolId: pool.poolId,
-            share: pool['share'],
-            type: 'weekly',
-          });
+      while (toTimestamp > genesisTimestamp) {
+        const fromTimestamp = toTimestamp - 86400;
+        if (indexedTimestamp.includes(fromTimestamp)) {
+          toTimestamp -= 86400;
+          ++totalIndexed;
+          continue;
         }
+
+        const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
+          null, fromTimestamp, toTimestamp);
+        if (blockStats.blockCount === 0) { // We are done indexing, no blocks left
+          break;
+        }
+
+        let lastBlockHashrate = 0;
+        lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
+          blockStats.lastBlockHeight);
+
+        if (totalIndexed > 7 && totalIndexed % 7 === 0 && !indexedTimestamp.includes(fromTimestamp + 1)) { // Save weekly pools hashrate
+          logger.debug("Indexing weekly hashrates for mining pools");
+          let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp - 604800, fromTimestamp);
+          const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
+          pools = pools.map((pool: any) => {
+            pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
+            pool.share = (pool.blockCount / totalBlocks);
+            return pool;
+          });
+
+          for (const pool of pools) {
+            hashrates.push({
+              hashrateTimestamp: fromTimestamp + 1,
+              avgHashrate: pool['hashrate'],
+              poolId: pool.poolId,
+              share: pool['share'],
+              type: 'weekly',
+            });
+          }
+        }
+
+        hashrates.push({
+          hashrateTimestamp: fromTimestamp,
+          avgHashrate: lastBlockHashrate,
+          poolId: null,
+          share: 1,
+          type: 'daily',
+        });
+
+        if (hashrates.length > 10) {
+          await HashratesRepository.$saveHashrates(hashrates);
+          hashrates.length = 0;
+        }
+
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+        if (elapsedSeconds > 5) {
+          const daysPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2);
+          const formattedDate = new Date(fromTimestamp * 1000).toUTCString();
+          const daysLeft = Math.round(totalDayIndexed - totalIndexed);
+          logger.debug(`Getting hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ~${daysLeft} days left to index`);
+          startedAt = new Date().getTime() / 1000;
+          indexedThisRun = 0;
+        }
+
+        toTimestamp -= 86400;
+        ++indexedThisRun;
+        ++totalIndexed;
       }
 
-      hashrates.push({
-        hashrateTimestamp: fromTimestamp,
-        avgHashrate: lastBlockHashrate,
-        poolId: null,
-        share: 1,
-        type: 'daily',
-      });
+      // Add genesis block manually
+      if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) {
+        hashrates.push({
+          hashrateTimestamp: genesisTimestamp,
+          avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
+          poolId: null,
+          type: 'daily',
+        });
+      }
 
-      if (hashrates.length > 10) {
+      if (hashrates.length > 0) {
         await HashratesRepository.$saveHashrates(hashrates);
-        hashrates.length = 0;
       }
+      await HashratesRepository.$setLatestRunTimestamp();
+      this.hashrateIndexingStarted = false;
 
-      const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
-      if (elapsedSeconds > 5) {
-        const daysPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2);
-        const formattedDate = new Date(fromTimestamp * 1000).toUTCString();
-        const daysLeft = Math.round(totalDayIndexed - totalIndexed);
-        logger.debug(`Getting hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ~${daysLeft} days left to index`);
-        startedAt = new Date().getTime() / 1000;
-        indexedThisRun = 0;
-      }
-
-      toTimestamp -= 86400;
-      ++indexedThisRun;
-      ++totalIndexed;
+      logger.info(`Hashrates indexing completed`);
+    } catch (e) {
+      this.hashrateIndexingStarted = false;
+      throw e;
     }
-
-    // Add genesis block manually
-    if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) {
-      hashrates.push({
-        hashrateTimestamp: genesisTimestamp,
-        avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
-        poolId: null,
-        type: 'daily',
-      });
-    }
-
-    if (hashrates.length > 0) {
-      await HashratesRepository.$saveHashrates(hashrates);
-    }
-    await HashratesRepository.$setLatestRunTimestamp();
-    this.hashrateIndexingStarted = false;
-
-    logger.info(`Hashrates indexing completed`);
   }
 }
 

backend/src/index.ts

@@ -96,8 +96,8 @@ class Server {
           await Common.sleep(5000);
           await databaseMigration.$truncateIndexedData(tables);
         }
-        await this.$resetHashratesIndexingState();
         await databaseMigration.$initializeOrMigrateDatabase();
+        await this.$resetHashratesIndexingState();
         await poolsParser.migratePoolsJson();
       } catch (e) {
         throw new Error(e instanceof Error ? e.message : 'Error');
@@ -167,7 +167,7 @@ class Server {
   }
 
   async $resetHashratesIndexingState() {
-    return await HashratesRepository.$setLatestRunTimestamp(0);    
+    return await HashratesRepository.$setLatestRunTimestamp(0);
   }
 
   async $runIndexingWhenReady() {

backend/src/repositories/HashratesRepository.ts

@@ -22,6 +22,7 @@ class HashratesRepository {
       await connection.query(query);
     } catch (e: any) {
       logger.err('$saveHashrateInDatabase() error' + (e instanceof Error ? e.message : e));
+      throw e;
     }
 
     connection.release();