Merge pull request #1596 from mempool/nymkappa/feature/indexing-progress

Create indexing progress sticky notification
2024-12-27 08:44:26 +01:00 · 2022-05-10 23:02:48 +09:00 · 2022-05-10 23:02:48 +09:00 · 61a054b40f
commit 61a054b40f
parent 11cdbb3118 df177e0ea7
23 changed files with 155 additions and 40 deletions
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@ -221,9 +221,10 @@ class Blocks {
      const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);

      logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
+      loadingIndicators.setProgress('block-indexing', 0);

      const chunkSize = 10000;
-      let totaIndexed = await blocksRepository.$blockCount(null, null);
+      let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex);
      let indexedThisRun = 0;
      let newlyIndexed = 0;
      const startedAt = new Date().getTime() / 1000;
@ -246,16 +247,17 @@ class Blocks {
            break;
          }
          ++indexedThisRun;
-          ++totaIndexed;
+          ++totalIndexed;
          const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
          if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
            const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
            const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
-            const progress = Math.round(totaIndexed / indexingBlockAmount * 100);
-            const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds);
-            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
+            const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
+            const timeLeft = Math.round((indexingBlockAmount - totalIndexed) / blockPerSeconds);
+            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
            timer = new Date().getTime() / 1000;
            indexedThisRun = 0;
+            loadingIndicators.setProgress('block-indexing', progress, false);
          }
          const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
          const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
@ -269,9 +271,11 @@ class Blocks {
        currentBlockHeight -= chunkSize;
      }
      logger.info(`Indexed ${newlyIndexed} blocks`);
+      loadingIndicators.setProgress('block-indexing', 100);
    } catch (e) {
      logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));
      this.blockIndexingStarted = false;
+      loadingIndicators.setProgress('block-indexing', 100);
      return;
    }

--- a/backend/src/api/loading-indicators.ts
+++ b/backend/src/api/loading-indicators.ts
@ -12,8 +12,8 @@ class LoadingIndicators {
    this.progressChangedCallback = fn;
  }

-  public setProgress(name: string, progressPercent: number) {
-    const newProgress = Math.round(progressPercent);
+  public setProgress(name: string, progressPercent: number, rounded: boolean = true) {
+    const newProgress = rounded === true ? Math.round(progressPercent) : progressPercent;
    if (newProgress >= 100) {
      delete this.loadingIndicators[name];
    } else {
--- a/backend/src/api/mining.ts
+++ b/backend/src/api/mining.ts
@ -6,6 +6,7 @@ import bitcoinClient from './bitcoin/bitcoin-client';
 import logger from '../logger';
 import blocks from './blocks';
 import { Common } from './common';
+import loadingIndicators from './loading-indicators';

 class Mining {
  hashrateIndexingStarted = false;
@ -131,7 +132,7 @@ class Mining {
   * [INDEXING] Generate weekly mining pool hashrate history
   */
  public async $generatePoolHashrateHistory(): Promise<void> {
-    if (!blocks.blockIndexingCompleted || this.weeklyHashrateIndexingStarted) {
+    if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted || this.weeklyHashrateIndexingStarted) {
      return;
    }

@ -167,7 +168,10 @@ class Mining {
      let indexedThisRun = 0;
      let totalIndexed = 0;
      let newlyIndexed = 0;
-      let startedAt = new Date().getTime();
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 0);

      while (toTimestamp > genesisTimestamp) {
        const fromTimestamp = toTimestamp - 604800000;
@ -214,14 +218,17 @@ class Mining {
        await HashratesRepository.$saveHashrates(hashrates);
        hashrates.length = 0;

-        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime()) - startedAt)) / 1000;
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
        if (elapsedSeconds > 1) {
-          const weeksPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2);
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+          const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100;
+          const timeLeft = Math.round((totalWeekIndexed - totalIndexed) / weeksPerSeconds);
          const formattedDate = new Date(fromTimestamp).toUTCString();
-          const weeksLeft = Math.round(totalWeekIndexed - totalIndexed);
-          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds} weeks/sec | ~${weeksLeft} weeks left to index`);
-          startedAt = new Date().getTime();
+          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
+          timer = new Date().getTime() / 1000;
          indexedThisRun = 0;
+          loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false);
        }

        toTimestamp -= 604800000;
@ -233,7 +240,9 @@ class Mining {
      if (newlyIndexed > 0) {
        logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`);
      }
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
    } catch (e) {
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
      this.weeklyHashrateIndexingStarted = false;
      throw e;
    }
@ -273,7 +282,10 @@ class Mining {
      let indexedThisRun = 0;
      let totalIndexed = 0;
      let newlyIndexed = 0;
-      let startedAt = new Date().getTime();
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
+      loadingIndicators.setProgress('daily-hashrate-indexing', 0);

      while (toTimestamp > genesisTimestamp) {
        const fromTimestamp = toTimestamp - 86400000;
@ -312,15 +324,17 @@ class Mining {
          hashrates.length = 0;
        }

-        const elapsedSeconds = Math.max(1, Math.round(new Date().getTime() - startedAt)) / 1000;
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
        if (elapsedSeconds > 1) {
-          const daysPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2);
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+          const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100;
+          const timeLeft = Math.round((totalDayIndexed - totalIndexed) / daysPerSeconds);
          const formattedDate = new Date(fromTimestamp).toUTCString();
-          const daysLeft = Math.round(totalDayIndexed - totalIndexed);
-          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ` +
-            `~${daysLeft} days left to index`);
-          startedAt = new Date().getTime();
+          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
+          timer = new Date().getTime() / 1000;
          indexedThisRun = 0;
+          loadingIndicators.setProgress('daily-hashrate-indexing', progress);
        }

        toTimestamp -= 86400000;
@ -346,7 +360,9 @@ class Mining {
      if (newlyIndexed > 0) {
        logger.info(`Indexed ${newlyIndexed} day of network hashrate`);
      }
+      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
    } catch (e) {
+      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
      this.hashrateIndexingStarted = false;
      throw e;
    }
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@ -188,6 +188,24 @@ class BlocksRepository {
    }
  }

+  /**
+   * Get blocks count for a period
+   */
+   public async $blockCountBetweenHeight(startHeight: number, endHeight: number): Promise<number> {
+    const params: any[] = [];
+    let query = `SELECT count(height) as blockCount
+      FROM blocks
+      WHERE height <= ${startHeight} AND height >= ${endHeight}`;
+
+    try {
+      const [rows] = await DB.query(query, params);
+      return <number>rows[0].blockCount;
+    } catch (e) {
+      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
  /**
   * Get the oldest indexed block
   */
--- a/frontend/src/app/app.module.ts
+++ b/frontend/src/app/app.module.ts
@ -76,6 +76,8 @@ import { DataCyDirective } from './data-cy.directive';
 import { BlockFeesGraphComponent } from './components/block-fees-graph/block-fees-graph.component';
 import { BlockRewardsGraphComponent } from './components/block-rewards-graph/block-rewards-graph.component';
 import { BlockFeeRatesGraphComponent } from './components/block-fee-rates-graph/block-fee-rates-graph.component';
+import { LoadingIndicatorComponent } from './components/loading-indicator/loading-indicator.component';
+import { IndexingProgressComponent } from './components/indexing-progress/indexing-progress.component';

@NgModule({
  declarations: [
@ -132,6 +134,8 @@ import { BlockFeeRatesGraphComponent } from './components/block-fee-rates-graph/
    BlockFeesGraphComponent,
    BlockRewardsGraphComponent,
    BlockFeeRatesGraphComponent,
+    LoadingIndicatorComponent,
+    IndexingProgressComponent,
  ],
  imports: [
    BrowserModule.withServerTransition({ appId: 'serverApp' }),
--- a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html
+++ b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html
@ -1,3 +1,5 @@
+<app-indexing-progress></app-indexing-progress>
+
 <div class="full-container">
  <div class="card-header mb-0 mb-md-4">
    <span i18n="mining.block-fee-rates">Block fee rates</span>
--- a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.html
+++ b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.html
@ -1,3 +1,5 @@
+<app-indexing-progress></app-indexing-progress>
+
 <div class="full-container">
  <div class="card-header mb-0 mb-md-4">
    <span i18n="mining.block-fees">Block fees</span>
--- a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html
+++ b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html
@ -1,3 +1,5 @@
+<app-indexing-progress></app-indexing-progress>
+
 <div class="full-container">

  <div class="card-header mb-0 mb-md-4">
--- a/frontend/src/app/components/blocks-list/blocks-list.component.html
+++ b/frontend/src/app/components/blocks-list/blocks-list.component.html
@ -1,3 +1,5 @@
+<app-indexing-progress *ngIf="!widget"></app-indexing-progress>
+
 <div class="container-xl" [class]="widget ? 'widget' : 'full-height'">
  <h1 *ngIf="!widget" class="float-left" i18n="latest-blocks.blocks">Blocks</h1>

--- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html
+++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html
@ -1,3 +1,5 @@
+<app-indexing-progress *ngIf="!widget"></app-indexing-progress>
+
 <div [class]="widget === false ? 'full-container' : ''">

  <div *ngIf="widget">
--- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts
+++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts
@ -136,16 +136,12 @@ export class HashrateChartComponent implements OnInit {
  prepareChartOptions(data) {
    let title: object;
    if (data.hashrates.length === 0) {
-      const lastBlock = new Date(data.timestamp * 1000);
-      const dd = String(lastBlock.getDate()).padStart(2, '0');
-      const mm = String(lastBlock.getMonth() + 1).padStart(2, '0'); // January is 0!
-      const yyyy = lastBlock.getFullYear();
      title = {
        textStyle: {
          color: 'grey',
          fontSize: 15
        },
-        text: `Indexing in progess - ${yyyy}-${mm}-${dd}`,
+        text: `Indexing in progess`,
        left: 'center',
        top: 'center'
      };
--- a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.html
+++ b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.html
@ -1,3 +1,5 @@
+<app-indexing-progress></app-indexing-progress>
+
 <div class="full-container">

  <div class="card-header  mb-0 mb-md-4">
--- a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts
+++ b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts
@ -150,16 +150,12 @@ export class HashrateChartPoolsComponent implements OnInit {
  prepareChartOptions(data) {
    let title: object;
    if (data.series.length === 0) {
-      const lastBlock = new Date(data.timestamp * 1000);
-      const dd = String(lastBlock.getDate()).padStart(2, '0');
-      const mm = String(lastBlock.getMonth() + 1).padStart(2, '0'); // January is 0!
-      const yyyy = lastBlock.getFullYear();
      title = {
        textStyle: {
          color: 'grey',
          fontSize: 15
        },
-        text: `Indexing in progess - ${yyyy}-${mm}-${dd}`,
+        text: `Indexing in progess`,
        left: 'center',
        top: 'center',
      };
--- a/frontend/src/app/components/indexing-progress/indexing-progress.component.html
+++ b/frontend/src/app/components/indexing-progress/indexing-progress.component.html
@ -0,0 +1,3 @@
+<app-loading-indicator [name]="'block-indexing'" i18n-label label="Indexing blocks"></app-loading-indicator>
+<app-loading-indicator [name]="'daily-hashrate-indexing'" i18n-label label="Indexing network hashrate"></app-loading-indicator>
+<app-loading-indicator [name]="'weekly-hashrate-indexing'" i18n-label label="Indexing pools hashrate"></app-loading-indicator>
--- a/frontend/src/app/components/indexing-progress/indexing-progress.component.ts
+++ b/frontend/src/app/components/indexing-progress/indexing-progress.component.ts
@ -0,0 +1,14 @@
+import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
+
+@Component({
+  selector: 'app-indexing-progress',
+  templateUrl: './indexing-progress.component.html',
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class IndexingProgressComponent implements OnInit {
+  constructor(
+  ) {}
+
+  ngOnInit() {
+  }
+}
--- a/frontend/src/app/components/loading-indicator/loading-indicator.component.html
+++ b/frontend/src/app/components/loading-indicator/loading-indicator.component.html
@ -0,0 +1,3 @@
+<div *ngIf="this.indexingProgress$ | async as progress" class="sticky-loading">
+  <span *ngIf="progress >= 0" class="mr-auto badge badge-pill badge-warning">{{ this.label }} ({{ progress }}%)</span>
+</div>
--- a/frontend/src/app/components/loading-indicator/loading-indicator.component.scss
+++ b/frontend/src/app/components/loading-indicator/loading-indicator.component.scss
@ -0,0 +1,18 @@
+.sticky-loading {
+  position: absolute;
+  right: 10px;
+  z-index: 100;
+  font-size: 14px;
+  @media (width >= 992px) {
+    left: 32px;
+    top: 55px;
+  }
+  @media (576px <= width < 992px ) {
+    left: 18px;
+    top: 55px;
+  }
+  @media (width <= 575px) {
+    left: 18px;
+    top: 100px;
+  }
+}
--- a/frontend/src/app/components/loading-indicator/loading-indicator.component.ts
+++ b/frontend/src/app/components/loading-indicator/loading-indicator.component.ts
@ -0,0 +1,30 @@
+import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { StateService } from 'src/app/services/state.service';
+import { WebsocketService } from 'src/app/services/websocket.service';
+
+@Component({
+  selector: 'app-loading-indicator',
+  templateUrl: './loading-indicator.component.html',
+  styleUrls: ['./loading-indicator.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class LoadingIndicatorComponent implements OnInit {
+  @Input() name: string;
+  @Input() label: string;
+
+  public indexingProgress$: Observable<number>;
+
+  constructor(
+    private stateService: StateService,
+    private websocketService: WebsocketService
+  ) {}
+
+  ngOnInit() {
+    this.indexingProgress$ = this.stateService.loadingIndicators$
+      .pipe(
+        map((indicators) => indicators[this.name] ?? -1)
+      );
+  }
+}
--- a/frontend/src/app/components/mining-dashboard/mining-dashboard.component.html
+++ b/frontend/src/app/components/mining-dashboard/mining-dashboard.component.html
@ -1,3 +1,5 @@
+<app-indexing-progress></app-indexing-progress>
+
 <div class="container-xl dashboard-container">

  <div class="row row-cols-1 row-cols-md-2">
--- a/frontend/src/app/components/mining-dashboard/mining-dashboard.component.ts
+++ b/frontend/src/app/components/mining-dashboard/mining-dashboard.component.ts
@ -1,8 +1,5 @@
 import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
-import { map } from 'rxjs/operators';
 import { SeoService } from 'src/app/services/seo.service';
-import { StateService } from 'src/app/services/state.service';
-import { Observable } from 'rxjs';
 import { WebsocketService } from 'src/app/services/websocket.service';

@Component({
@ -12,8 +9,6 @@ import { WebsocketService } from 'src/app/services/websocket.service';
  changeDetection: ChangeDetectionStrategy.OnPush,
 })
 export class MiningDashboardComponent implements OnInit {
-  private blocks = [];
-
  constructor(
    private seoService: SeoService,
    private websocketService: WebsocketService,
--- a/frontend/src/app/components/pool-ranking/pool-ranking.component.html
+++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.html
@ -1,3 +1,5 @@
+<app-indexing-progress *ngIf="!widget"></app-indexing-progress>
+
 <div [class]="widget === false ? 'full-container' : ''">

  <div *ngIf="widget">
--- a/frontend/src/app/components/pool/pool.component.html
+++ b/frontend/src/app/components/pool/pool.component.html
@ -1,3 +1,5 @@
+<app-indexing-progress></app-indexing-progress>
+
 <div class="container-xl">

  <!-- Pool overview -->
--- a/frontend/src/app/components/pool/pool.component.ts
+++ b/frontend/src/app/components/pool/pool.component.ts
@ -111,7 +111,7 @@ export class PoolComponent implements OnInit {
          color: 'grey',
          fontSize: 15
        },
-        text: `No data`,
+        text: `Indexing in progress`,
        left: 'center',
        top: 'center'
      };
@ -164,14 +164,14 @@ export class PoolComponent implements OnInit {
          `;
        }.bind(this)
      },
-      xAxis: {
+      xAxis: data.length === 0 ? undefined : {
        type: 'time',
        splitNumber: (this.isMobile()) ? 5 : 10,
        axisLabel: {
          hideOverlap: true,
        }
      },
-      yAxis: [
+      yAxis: data.length === 0 ? undefined : [
        {
          min: (value) => {
            return value.min * 0.9;
@ -190,7 +190,7 @@ export class PoolComponent implements OnInit {
          }
        },
      ],
-      series: [
+      series: data.length === 0 ? undefined : [
        {
          zlevel: 0,
          name: 'Hashrate',