Merge pull request #1596 from mempool/nymkappa/feature/indexing-progress

Create indexing progress sticky notification

backend/src/api/blocks.ts

@@ -221,9 +221,10 @@ class Blocks {
       const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
 
       logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
+      loadingIndicators.setProgress('block-indexing', 0);
 
       const chunkSize = 10000;
-      let totaIndexed = await blocksRepository.$blockCount(null, null);
+      let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex);
       let indexedThisRun = 0;
       let newlyIndexed = 0;
       const startedAt = new Date().getTime() / 1000;
@@ -246,16 +247,17 @@ class Blocks {
             break;
           }
           ++indexedThisRun;
-          ++totaIndexed;
+          ++totalIndexed;
           const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
           if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
             const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
             const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
-            const progress = Math.round(totaIndexed / indexingBlockAmount * 100);
+            const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
-            const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds);
+            const timeLeft = Math.round((indexingBlockAmount - totalIndexed) / blockPerSeconds);
-            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
+            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
             timer = new Date().getTime() / 1000;
             indexedThisRun = 0;
+            loadingIndicators.setProgress('block-indexing', progress, false);
           }
           const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
           const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
@@ -269,9 +271,11 @@ class Blocks {
         currentBlockHeight -= chunkSize;
       }
       logger.info(`Indexed ${newlyIndexed} blocks`);
+      loadingIndicators.setProgress('block-indexing', 100);
     } catch (e) {
       logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));
       this.blockIndexingStarted = false;
+      loadingIndicators.setProgress('block-indexing', 100);
       return;
     }
 

backend/src/api/loading-indicators.ts

@@ -12,8 +12,8 @@ class LoadingIndicators {
     this.progressChangedCallback = fn;
   }
 
-  public setProgress(name: string, progressPercent: number) {
+  public setProgress(name: string, progressPercent: number, rounded: boolean = true) {
-    const newProgress = Math.round(progressPercent);
+    const newProgress = rounded === true ? Math.round(progressPercent) : progressPercent;
     if (newProgress >= 100) {
       delete this.loadingIndicators[name];
     } else {

backend/src/api/mining.ts

@@ -6,6 +6,7 @@ import bitcoinClient from './bitcoin/bitcoin-client';
 import logger from '../logger';
 import blocks from './blocks';
 import { Common } from './common';
+import loadingIndicators from './loading-indicators';
 
 class Mining {
   hashrateIndexingStarted = false;
@@ -131,7 +132,7 @@ class Mining {
    * [INDEXING] Generate weekly mining pool hashrate history
    */
   public async $generatePoolHashrateHistory(): Promise<void> {
-    if (!blocks.blockIndexingCompleted || this.weeklyHashrateIndexingStarted) {
+    if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted || this.weeklyHashrateIndexingStarted) {
       return;
     }
 
@@ -167,7 +168,10 @@ class Mining {
       let indexedThisRun = 0;
       let totalIndexed = 0;
       let newlyIndexed = 0;
-      let startedAt = new Date().getTime();
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
 
       while (toTimestamp > genesisTimestamp) {
         const fromTimestamp = toTimestamp - 604800000;
@@ -214,14 +218,17 @@ class Mining {
         await HashratesRepository.$saveHashrates(hashrates);
         hashrates.length = 0;
 
-        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime()) - startedAt)) / 1000;
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
         if (elapsedSeconds > 1) {
-          const weeksPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2);
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+          const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100;
+          const timeLeft = Math.round((totalWeekIndexed - totalIndexed) / weeksPerSeconds);
           const formattedDate = new Date(fromTimestamp).toUTCString();
-          const weeksLeft = Math.round(totalWeekIndexed - totalIndexed);
+          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
-          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds} weeks/sec | ~${weeksLeft} weeks left to index`);
+          timer = new Date().getTime() / 1000;
-          startedAt = new Date().getTime();
           indexedThisRun = 0;
+          loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false);
         }
 
         toTimestamp -= 604800000;
@@ -233,7 +240,9 @@ class Mining {
       if (newlyIndexed > 0) {
         logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`);
       }
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
     } catch (e) {
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
       this.weeklyHashrateIndexingStarted = false;
       throw e;
     }
@@ -273,7 +282,10 @@ class Mining {
       let indexedThisRun = 0;
       let totalIndexed = 0;
       let newlyIndexed = 0;
-      let startedAt = new Date().getTime();
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
+      loadingIndicators.setProgress('daily-hashrate-indexing', 0);
 
       while (toTimestamp > genesisTimestamp) {
         const fromTimestamp = toTimestamp - 86400000;
@@ -312,15 +324,17 @@ class Mining {
           hashrates.length = 0;
         }
 
-        const elapsedSeconds = Math.max(1, Math.round(new Date().getTime() - startedAt)) / 1000;
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
         if (elapsedSeconds > 1) {
-          const daysPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2);
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+          const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100;
+          const timeLeft = Math.round((totalDayIndexed - totalIndexed) / daysPerSeconds);
           const formattedDate = new Date(fromTimestamp).toUTCString();
-          const daysLeft = Math.round(totalDayIndexed - totalIndexed);
+          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
-          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ` +
+          timer = new Date().getTime() / 1000;
-            `~${daysLeft} days left to index`);
-          startedAt = new Date().getTime();
           indexedThisRun = 0;
+          loadingIndicators.setProgress('daily-hashrate-indexing', progress);
         }
 
         toTimestamp -= 86400000;
@@ -346,7 +360,9 @@ class Mining {
       if (newlyIndexed > 0) {
         logger.info(`Indexed ${newlyIndexed} day of network hashrate`);
       }
+      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
     } catch (e) {
+      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
       this.hashrateIndexingStarted = false;
       throw e;
     }

backend/src/repositories/BlocksRepository.ts

@@ -188,6 +188,24 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Get blocks count for a period
+   */
+   public async $blockCountBetweenHeight(startHeight: number, endHeight: number): Promise<number> {
+    const params: any[] = [];
+    let query = `SELECT count(height) as blockCount
+      FROM blocks
+      WHERE height <= ${startHeight} AND height >= ${endHeight}`;
+
+    try {
+      const [rows] = await DB.query(query, params);
+      return <number>rows[0].blockCount;
+    } catch (e) {
+      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Get the oldest indexed block
    */

frontend/src/app/app.module.ts

@@ -76,6 +76,8 @@ import { DataCyDirective } from './data-cy.directive';
 import { BlockFeesGraphComponent } from './components/block-fees-graph/block-fees-graph.component';
 import { BlockRewardsGraphComponent } from './components/block-rewards-graph/block-rewards-graph.component';
 import { BlockFeeRatesGraphComponent } from './components/block-fee-rates-graph/block-fee-rates-graph.component';
+import { LoadingIndicatorComponent } from './components/loading-indicator/loading-indicator.component';
+import { IndexingProgressComponent } from './components/indexing-progress/indexing-progress.component';
 
 @NgModule({
   declarations: [
@@ -132,6 +134,8 @@ import { BlockFeeRatesGraphComponent } from './components/block-fee-rates-graph/
     BlockFeesGraphComponent,
     BlockRewardsGraphComponent,
     BlockFeeRatesGraphComponent,
+    LoadingIndicatorComponent,
+    IndexingProgressComponent,
   ],
   imports: [
     BrowserModule.withServerTransition({ appId: 'serverApp' }),

frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html

@@ -1,3 +1,5 @@
+<app-indexing-progress></app-indexing-progress>
+
 <div class="full-container">
   <div class="card-header mb-0 mb-md-4">
     <span i18n="mining.block-fee-rates">Block fee rates</span>

frontend/src/app/components/block-fees-graph/block-fees-graph.component.html

@@ -1,3 +1,5 @@
+<app-indexing-progress></app-indexing-progress>
+
 <div class="full-container">
   <div class="card-header mb-0 mb-md-4">
     <span i18n="mining.block-fees">Block fees</span>

frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html

@@ -1,3 +1,5 @@
+<app-indexing-progress></app-indexing-progress>
+
 <div class="full-container">
 
   <div class="card-header mb-0 mb-md-4">

frontend/src/app/components/blocks-list/blocks-list.component.html

@@ -1,3 +1,5 @@
+<app-indexing-progress *ngIf="!widget"></app-indexing-progress>
+
 <div class="container-xl" [class]="widget ? 'widget' : 'full-height'">
   <h1 *ngIf="!widget" class="float-left" i18n="latest-blocks.blocks">Blocks</h1>
 

frontend/src/app/components/hashrate-chart/hashrate-chart.component.html

@@ -1,3 +1,5 @@
+<app-indexing-progress *ngIf="!widget"></app-indexing-progress>
+
 <div [class]="widget === false ? 'full-container' : ''">
 
   <div *ngIf="widget">

frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts

@@ -136,16 +136,12 @@ export class HashrateChartComponent implements OnInit {
   prepareChartOptions(data) {
     let title: object;
     if (data.hashrates.length === 0) {
-      const lastBlock = new Date(data.timestamp * 1000);
-      const dd = String(lastBlock.getDate()).padStart(2, '0');
-      const mm = String(lastBlock.getMonth() + 1).padStart(2, '0'); // January is 0!
-      const yyyy = lastBlock.getFullYear();
       title = {
         textStyle: {
           color: 'grey',
           fontSize: 15
         },
-        text: `Indexing in progess - ${yyyy}-${mm}-${dd}`,
+        text: `Indexing in progess`,
         left: 'center',
         top: 'center'
       };

frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.html

@@ -1,3 +1,5 @@
+<app-indexing-progress></app-indexing-progress>
+
 <div class="full-container">
 
   <div class="card-header  mb-0 mb-md-4">

frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts

@@ -150,16 +150,12 @@ export class HashrateChartPoolsComponent implements OnInit {
   prepareChartOptions(data) {
     let title: object;
     if (data.series.length === 0) {
-      const lastBlock = new Date(data.timestamp * 1000);
-      const dd = String(lastBlock.getDate()).padStart(2, '0');
-      const mm = String(lastBlock.getMonth() + 1).padStart(2, '0'); // January is 0!
-      const yyyy = lastBlock.getFullYear();
       title = {
         textStyle: {
           color: 'grey',
           fontSize: 15
         },
-        text: `Indexing in progess - ${yyyy}-${mm}-${dd}`,
+        text: `Indexing in progess`,
         left: 'center',
         top: 'center',
       };

frontend/src/app/components/indexing-progress/indexing-progress.component.html

@@ -0,0 +1,3 @@
+<app-loading-indicator [name]="'block-indexing'" i18n-label label="Indexing blocks"></app-loading-indicator>
+<app-loading-indicator [name]="'daily-hashrate-indexing'" i18n-label label="Indexing network hashrate"></app-loading-indicator>
+<app-loading-indicator [name]="'weekly-hashrate-indexing'" i18n-label label="Indexing pools hashrate"></app-loading-indicator>

frontend/src/app/components/indexing-progress/indexing-progress.component.ts

@@ -0,0 +1,14 @@
+import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
+
+@Component({
+  selector: 'app-indexing-progress',
+  templateUrl: './indexing-progress.component.html',
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class IndexingProgressComponent implements OnInit {
+  constructor(
+  ) {}
+
+  ngOnInit() {
+  }
+}

frontend/src/app/components/loading-indicator/loading-indicator.component.html

@@ -0,0 +1,3 @@
+<div *ngIf="this.indexingProgress$ | async as progress" class="sticky-loading">
+  <span *ngIf="progress >= 0" class="mr-auto badge badge-pill badge-warning">{{ this.label }} ({{ progress }}%)</span>
+</div>

frontend/src/app/components/loading-indicator/loading-indicator.component.scss

@@ -0,0 +1,18 @@
+.sticky-loading {
+  position: absolute;
+  right: 10px;
+  z-index: 100;
+  font-size: 14px;
+  @media (width >= 992px) {
+    left: 32px;
+    top: 55px;
+  }
+  @media (576px <= width < 992px ) {
+    left: 18px;
+    top: 55px;
+  }
+  @media (width <= 575px) {
+    left: 18px;
+    top: 100px;
+  }
+}

frontend/src/app/components/loading-indicator/loading-indicator.component.ts

@@ -0,0 +1,30 @@
+import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { StateService } from 'src/app/services/state.service';
+import { WebsocketService } from 'src/app/services/websocket.service';
+
+@Component({
+  selector: 'app-loading-indicator',
+  templateUrl: './loading-indicator.component.html',
+  styleUrls: ['./loading-indicator.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class LoadingIndicatorComponent implements OnInit {
+  @Input() name: string;
+  @Input() label: string;
+
+  public indexingProgress$: Observable<number>;
+
+  constructor(
+    private stateService: StateService,
+    private websocketService: WebsocketService
+  ) {}
+
+  ngOnInit() {
+    this.indexingProgress$ = this.stateService.loadingIndicators$
+      .pipe(
+        map((indicators) => indicators[this.name] ?? -1)
+      );
+  }
+}

frontend/src/app/components/mining-dashboard/mining-dashboard.component.html

@@ -1,3 +1,5 @@
+<app-indexing-progress></app-indexing-progress>
+
 <div class="container-xl dashboard-container">
 
   <div class="row row-cols-1 row-cols-md-2">

frontend/src/app/components/mining-dashboard/mining-dashboard.component.ts

@@ -1,8 +1,5 @@
 import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
-import { map } from 'rxjs/operators';
 import { SeoService } from 'src/app/services/seo.service';
-import { StateService } from 'src/app/services/state.service';
-import { Observable } from 'rxjs';
 import { WebsocketService } from 'src/app/services/websocket.service';
 
 @Component({
@@ -12,8 +9,6 @@ import { WebsocketService } from 'src/app/services/websocket.service';
   changeDetection: ChangeDetectionStrategy.OnPush,
 })
 export class MiningDashboardComponent implements OnInit {
-  private blocks = [];
-
   constructor(
     private seoService: SeoService,
     private websocketService: WebsocketService,

frontend/src/app/components/pool-ranking/pool-ranking.component.html

@@ -1,3 +1,5 @@
+<app-indexing-progress *ngIf="!widget"></app-indexing-progress>
+
 <div [class]="widget === false ? 'full-container' : ''">
 
   <div *ngIf="widget">

frontend/src/app/components/pool/pool.component.html

@@ -1,3 +1,5 @@
+<app-indexing-progress></app-indexing-progress>
+
 <div class="container-xl">
 
   <!-- Pool overview -->

frontend/src/app/components/pool/pool.component.ts

@@ -111,7 +111,7 @@ export class PoolComponent implements OnInit {
           color: 'grey',
           fontSize: 15
         },
-        text: `No data`,
+        text: `Indexing in progress`,
         left: 'center',
         top: 'center'
       };
@@ -164,14 +164,14 @@ export class PoolComponent implements OnInit {
           `;
         }.bind(this)
       },
-      xAxis: {
+      xAxis: data.length === 0 ? undefined : {
         type: 'time',
         splitNumber: (this.isMobile()) ? 5 : 10,
         axisLabel: {
           hideOverlap: true,
         }
       },
-      yAxis: [
+      yAxis: data.length === 0 ? undefined : [
         {
           min: (value) => {
             return value.min * 0.9;
@@ -190,7 +190,7 @@ export class PoolComponent implements OnInit {
           }
         },
       ],
-      series: [
+      series: data.length === 0 ? undefined : [
         {
           zlevel: 0,
           name: 'Hashrate',