From 38b37a3ee7d9bb4227bf70aca85cd8af62e23fd9 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 18 Feb 2022 15:26:15 +0900 Subject: [PATCH 01/19] Re-define sub mining routes properly and use router outlet --- frontend/src/app/app-routing.module.ts | 119 +++++++++++++----- frontend/src/app/app.module.ts | 4 + .../hashrate-chart.component.html | 1 + .../hashrate-chart.component.scss | 0 .../hashrate-chart.component.ts | 15 +++ .../mining-start/mining-start.component.html | 1 + .../mining-start/mining-start.component.ts | 14 +++ 7 files changed, 124 insertions(+), 30 deletions(-) create mode 100644 frontend/src/app/components/hashrate-chart/hashrate-chart.component.html create mode 100644 frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss create mode 100644 frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts create mode 100644 frontend/src/app/components/mining-start/mining-start.component.html create mode 100644 frontend/src/app/components/mining-start/mining-start.component.ts diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index f0aa73e3d..c8a1d98e6 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -29,6 +29,8 @@ import { AssetsComponent } from './components/assets/assets.component'; import { PoolComponent } from './components/pool/pool.component'; import { MiningDashboardComponent } from './components/mining-dashboard/mining-dashboard.component'; import { DifficultyChartComponent } from './components/difficulty-chart/difficulty-chart.component'; +import { HashrateChartComponent } from './components/hashrate-chart/hashrate-chart.component'; +import { MiningStartComponent } from './components/mining-start/mining-start.component'; let routes: Routes = [ { @@ -70,16 +72,35 @@ let routes: Routes = [ component: LatestBlocksComponent, }, { - path: 'mining/difficulty', - component: DifficultyChartComponent, - }, - { - path: 'mining/pools', - component: PoolRankingComponent, - }, - { - path: 'mining/pool/:poolId', - component: PoolComponent, + path: 'mining', + component: MiningStartComponent, + children: [ + { + path: 'difficulty', + component: DifficultyChartComponent, + }, + { + path: 'hashrate', + component: HashrateChartComponent, + }, + { + path: 'pools', + component: PoolRankingComponent, + }, + { + path: 'pool', + children: [ + { + path: ':poolId', + component: PoolComponent, + }, + { + path: ':poolId/hashrate', + component: HashrateChartComponent, + }, + ] + }, + ] }, { path: 'graphs', @@ -170,16 +191,35 @@ let routes: Routes = [ component: LatestBlocksComponent, }, { - path: 'mining/difficulty', - component: DifficultyChartComponent, - }, - { - path: 'mining/pools', - component: PoolRankingComponent, - }, - { - path: 'mining/pool/:poolId', - component: PoolComponent, + path: 'mining', + component: MiningStartComponent, + children: [ + { + path: 'difficulty', + component: DifficultyChartComponent, + }, + { + path: 'hashrate', + component: HashrateChartComponent, + }, + { + path: 'pools', + component: PoolRankingComponent, + }, + { + path: 'pool', + children: [ + { + path: ':poolId', + component: PoolComponent, + }, + { + path: ':poolId/hashrate', + component: HashrateChartComponent, + }, + ] + }, + ] }, { path: 'graphs', @@ -264,16 +304,35 @@ let routes: Routes = [ component: LatestBlocksComponent, }, { - path: 'mining/difficulty', - component: DifficultyChartComponent, - }, - { - path: 'mining/pools', - component: PoolRankingComponent, - }, - { - path: 'mining/pool/:poolId', - component: PoolComponent, + path: 'mining', + component: MiningStartComponent, + children: [ + { + path: 'difficulty', + component: DifficultyChartComponent, + }, + { + path: 'hashrate', + component: HashrateChartComponent, + }, + { + path: 'pools', + component: PoolRankingComponent, + }, + { + path: 'pool', + children: [ + { + path: ':poolId', + component: PoolComponent, + }, + { + path: ':poolId/hashrate', + component: HashrateChartComponent, + }, + ] + }, + ] }, { path: 'graphs', diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 4fe24ceb8..9e8fea464 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -71,6 +71,8 @@ import { AssetGroupComponent } from './components/assets/asset-group/asset-group import { AssetCirculationComponent } from './components/asset-circulation/asset-circulation.component'; import { MiningDashboardComponent } from './components/mining-dashboard/mining-dashboard.component'; import { DifficultyChartComponent } from './components/difficulty-chart/difficulty-chart.component'; +import { HashrateChartComponent } from './components/hashrate-chart/hashrate-chart.component'; +import { MiningStartComponent } from './components/mining-start/mining-start.component'; @NgModule({ declarations: [ @@ -124,6 +126,8 @@ import { DifficultyChartComponent } from './components/difficulty-chart/difficul AssetCirculationComponent, MiningDashboardComponent, DifficultyChartComponent, + HashrateChartComponent, + MiningStartComponent, ], imports: [ BrowserModule.withServerTransition({ appId: 'serverApp' }), diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html new file mode 100644 index 000000000..e55205844 --- /dev/null +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html @@ -0,0 +1 @@ +

hashrate-chart works!

diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss new file mode 100644 index 000000000..e69de29bb diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts new file mode 100644 index 000000000..cfbb6ba31 --- /dev/null +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts @@ -0,0 +1,15 @@ +import { Component, OnInit } from '@angular/core'; + +@Component({ + selector: 'app-hashrate-chart', + templateUrl: './hashrate-chart.component.html', + styleUrls: ['./hashrate-chart.component.scss'] +}) +export class HashrateChartComponent implements OnInit { + + constructor() { } + + ngOnInit(): void { + } + +} diff --git a/frontend/src/app/components/mining-start/mining-start.component.html b/frontend/src/app/components/mining-start/mining-start.component.html new file mode 100644 index 000000000..0680b43f9 --- /dev/null +++ b/frontend/src/app/components/mining-start/mining-start.component.html @@ -0,0 +1 @@ + diff --git a/frontend/src/app/components/mining-start/mining-start.component.ts b/frontend/src/app/components/mining-start/mining-start.component.ts new file mode 100644 index 000000000..6850cfa54 --- /dev/null +++ b/frontend/src/app/components/mining-start/mining-start.component.ts @@ -0,0 +1,14 @@ +import { Component, OnInit } from '@angular/core'; + +@Component({ + selector: 'app-mining-start', + templateUrl: './mining-start.component.html', +}) +export class MiningStartComponent implements OnInit { + + constructor() { } + + ngOnInit(): void { + } + +} From 6fe8f6fa1eb95fffd616e87c1dd3333bd30a76b0 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 19 Feb 2022 20:45:02 +0900 Subject: [PATCH 02/19] Generate daily average hashrate data --- backend/src/api/blocks.ts | 5 +- backend/src/api/database-migration.ts | 19 ++++++- backend/src/api/mining.ts | 52 ++++++++++++++++++- backend/src/index.ts | 11 +++- backend/src/repositories/BlocksRepository.ts | 34 ++++++++++++ .../src/repositories/HashratesRepository.ts | 42 +++++++++++++++ backend/src/routes.ts | 1 - 7 files changed, 156 insertions(+), 8 deletions(-) create mode 100644 backend/src/repositories/HashratesRepository.ts diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 1452b6fc8..483fd3f3e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -170,10 +170,7 @@ class Blocks { * Index all blocks metadata for the mining dashboard */ public async $generateBlockDatabase() { - if (this.blockIndexingStarted === true || - !Common.indexingEnabled() || - memPool.hasPriority() - ) { + if (this.blockIndexingStarted) { return; } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index b44585580..154068164 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,7 +6,7 @@ import logger from '../logger'; const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 6; + private static currentVersion = 7; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -116,6 +116,12 @@ class DatabaseMigration { await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""'); await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL'); } + + if (databaseSchemaVersion < 7 && isBitcoin === true) { + await this.$executeQuery(connection, 'DROP table IF EXISTS hashrates;'); + await this.$executeQuery(connection, this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates')); + } + connection.release(); } catch (e) { connection.release(); @@ -398,6 +404,17 @@ class DatabaseMigration { FOREIGN KEY (pool_id) REFERENCES pools (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + + private getCreateDailyStatsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS hashrates ( + hashrate_timestamp timestamp NOT NULL, + avg_hashrate double unsigned DEFAULT '0', + pool_id smallint unsigned NULL, + PRIMARY KEY (hashrate_timestamp), + INDEX (pool_id), + FOREIGN KEY (pool_id) REFERENCES pools (id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } } export default new DatabaseMigration(); diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index beca52893..fc13c2f5e 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -1,9 +1,13 @@ import { PoolInfo, PoolStats } from '../mempool.interfaces'; import BlocksRepository, { EmptyBlocks } from '../repositories/BlocksRepository'; import PoolsRepository from '../repositories/PoolsRepository'; +import HashratesRepository from '../repositories/HashratesRepository'; import bitcoinClient from './bitcoin/bitcoin-client'; +import logger from '../logger'; class Mining { + hashrateIndexingStarted = false; + constructor() { } @@ -45,7 +49,7 @@ class Mining { poolsStatistics['blockCount'] = blockCount; const blockHeightTip = await bitcoinClient.getBlockCount(); - const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(120, blockHeightTip); + const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(144, blockHeightTip); poolsStatistics['lastEstimatedHashrate'] = lastBlockHashrate; return poolsStatistics; @@ -82,6 +86,52 @@ class Mining { oldestIndexedBlockTimestamp: oldestBlock.getTime(), } } + + /** + * + */ + public async $generateNetworkHashrateHistory() : Promise { + if (this.hashrateIndexingStarted) { + return; + } + this.hashrateIndexingStarted = true; + + const totalIndexed = await BlocksRepository.$blockCount(null, null); + const indexedTimestamp = await HashratesRepository.$getAllTimestamp(); + + const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f + const lastMidnight = new Date(); + lastMidnight.setUTCHours(0); lastMidnight.setUTCMinutes(0); lastMidnight.setUTCSeconds(0); lastMidnight.setUTCMilliseconds(0); + let toTimestamp = Math.round(lastMidnight.getTime() / 1000); + + while (toTimestamp > genesisTimestamp) { + const fromTimestamp = toTimestamp - 86400; + if (indexedTimestamp.includes(fromTimestamp)) { + toTimestamp -= 86400; + continue; + } + + const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp( + null, fromTimestamp, toTimestamp + ); + let lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, blockStats.lastBlockHeight); + + if (toTimestamp % 864000 === 0) { + const progress = Math.round((totalIndexed - blockStats.lastBlockHeight) / totalIndexed * 100); + const formattedDate = new Date(fromTimestamp * 1000).toUTCString(); + logger.debug(`Counting blocks and hashrate for ${formattedDate}. Progress: ${progress}%`); + } + + await HashratesRepository.$saveDailyStat({ + hashrateTimestamp: fromTimestamp, + avgHashrate: lastBlockHashrate, + poolId: null, + }); + + toTimestamp -= 86400; + } + } + } export default new Mining(); diff --git a/backend/src/index.ts b/backend/src/index.ts index 23c70f59d..d09196a45 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -26,6 +26,7 @@ import poolsParser from './api/pools-parser'; import syncAssets from './sync-assets'; import icons from './api/liquid/icons'; import { Common } from './api/common'; +import mining from './api/mining'; class Server { private wss: WebSocket.Server | undefined; @@ -138,7 +139,7 @@ class Server { } await blocks.$updateBlocks(); await memPool.$updateMempool(); - blocks.$generateBlockDatabase(); + this.runIndexingWhenReady(); setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); this.currentBackendRetryInterval = 5; @@ -157,6 +158,14 @@ class Server { } } + async runIndexingWhenReady() { + if (!Common.indexingEnabled() || mempool.hasPriority()) { + return; + } + await blocks.$generateBlockDatabase(); + await mining.$generateNetworkHashrateHistory(); + } + setUpWebsocketHandling() { if (this.wss) { websocketHandler.setWebsocketServer(this.wss); diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index ac0ea25bc..937320a3a 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -149,6 +149,40 @@ class BlocksRepository { return rows[0].blockCount; } + /** + * Get blocks count between two dates + * @param poolId + * @param from - The oldest timestamp + * @param to - The newest timestamp + * @returns + */ + public async $blockCountBetweenTimestamp(poolId: number | null, from: number, to: number): Promise { + const params: any[] = []; + let query = `SELECT + count(height) as blockCount, + max(height) as lastBlockHeight + FROM blocks`; + + if (poolId) { + query += ` WHERE pool_id = ?`; + params.push(poolId); + } + + if (poolId) { + query += ` AND`; + } else { + query += ` WHERE`; + } + query += ` UNIX_TIMESTAMP(blockTimestamp) BETWEEN '${from}' AND '${to}'`; + + // logger.debug(query); + const connection = await DB.pool.getConnection(); + const [rows] = await connection.query(query, params); + connection.release(); + + return rows[0]; + } + /** * Get the oldest indexed block */ diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts new file mode 100644 index 000000000..f55700812 --- /dev/null +++ b/backend/src/repositories/HashratesRepository.ts @@ -0,0 +1,42 @@ +import { DB } from '../database'; +import logger from '../logger'; + +class HashratesRepository { + /** + * Save indexed block data in the database + */ + public async $saveDailyStat(dailyStat: any) { + const connection = await DB.pool.getConnection(); + + try { + const query = `INSERT INTO + hashrates(hashrate_timestamp, avg_hashrate, pool_id) + VALUE (FROM_UNIXTIME(?), ?, ?)`; + + const params: any[] = [ + dailyStat.hashrateTimestamp, dailyStat.avgHashrate, + dailyStat.poolId + ]; + + // logger.debug(query); + await connection.query(query, params); + } catch (e: any) { + logger.err('$saveHashrateInDatabase() error' + (e instanceof Error ? e.message : e)); + } + + connection.release(); + } + + /** + * Returns an array of all timestamp we've already indexed + */ + public async $getAllTimestamp(): Promise { + const connection = await DB.pool.getConnection(); + const [rows]: any[] = await connection.query(`SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp from hashrates`); + connection.release(); + + return rows.map(val => val.timestamp); + } +} + +export default new HashratesRepository(); diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 4a9cb1f8f..df5e7cb62 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -22,7 +22,6 @@ import elementsParser from './api/liquid/elements-parser'; import icons from './api/liquid/icons'; import miningStats from './api/mining'; import axios from 'axios'; -import PoolsRepository from './repositories/PoolsRepository'; import mining from './api/mining'; import BlocksRepository from './repositories/BlocksRepository'; From 358604ad85c1a805c8790c8eb230e024308529bf Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 19 Feb 2022 22:09:35 +0900 Subject: [PATCH 03/19] Added hashrate chart --- backend/src/api/mining.ts | 24 ++- backend/src/index.ts | 4 +- .../src/repositories/HashratesRepository.ts | 19 +- backend/src/routes.ts | 12 ++ .../hashrate-chart.component.html | 54 +++++- .../hashrate-chart.component.scss | 10 ++ .../hashrate-chart.component.ts | 166 +++++++++++++++++- frontend/src/app/services/api.service.ts | 7 + 8 files changed, 284 insertions(+), 12 deletions(-) diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index fc13c2f5e..6f90ab357 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -87,6 +87,19 @@ class Mining { } } + /** + * Return the historical hashrates and oldest indexed block timestamp + */ + public async $getHistoricalHashrates(interval: string | null): Promise { + const hashrates = await HashratesRepository.$get(interval); + const oldestBlock = new Date(await BlocksRepository.$oldestBlockTimestamp()); + + return { + hashrates: hashrates, + oldestIndexedBlockTimestamp: oldestBlock.getTime(), + } + } + /** * */ @@ -97,7 +110,7 @@ class Mining { this.hashrateIndexingStarted = true; const totalIndexed = await BlocksRepository.$blockCount(null, null); - const indexedTimestamp = await HashratesRepository.$getAllTimestamp(); + const indexedTimestamp = (await HashratesRepository.$get(null)).map(hashrate => hashrate.timestamp); const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f const lastMidnight = new Date(); @@ -114,7 +127,12 @@ class Mining { const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp( null, fromTimestamp, toTimestamp ); - let lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, blockStats.lastBlockHeight); + + let lastBlockHashrate = 0; + if (blockStats.blockCount > 0) { + lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, + blockStats.lastBlockHeight); + } if (toTimestamp % 864000 === 0) { const progress = Math.round((totalIndexed - blockStats.lastBlockHeight) / totalIndexed * 100); @@ -130,6 +148,8 @@ class Mining { toTimestamp -= 86400; } + + logger.info(`Hashrates indexing completed`); } } diff --git a/backend/src/index.ts b/backend/src/index.ts index d09196a45..4fe66bc72 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -285,7 +285,9 @@ class Server { .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:poolId', routes.$getPool) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:poolId/:interval', routes.$getPool) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty', routes.$getHistoricalDifficulty) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty/:interval', routes.$getHistoricalDifficulty); + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty/:interval', routes.$getHistoricalDifficulty) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate', routes.$getHistoricalHashrate) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', routes.$getHistoricalHashrate); } if (config.BISQ.ENABLED) { diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index f55700812..837569cd8 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -1,3 +1,4 @@ +import { Common } from '../api/common'; import { DB } from '../database'; import logger from '../logger'; @@ -30,12 +31,22 @@ class HashratesRepository { /** * Returns an array of all timestamp we've already indexed */ - public async $getAllTimestamp(): Promise { + public async $get(interval: string | null): Promise { + interval = Common.getSqlInterval(interval); + const connection = await DB.pool.getConnection(); - const [rows]: any[] = await connection.query(`SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp from hashrates`); + + let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate + FROM hashrates`; + + if (interval) { + query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`; + } + + const [rows]: any[] = await connection.query(query); connection.release(); - - return rows.map(val => val.timestamp); + + return rows; } } diff --git a/backend/src/routes.ts b/backend/src/routes.ts index df5e7cb62..1bf1c3434 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -586,6 +586,18 @@ class Routes { } } + public async $getHistoricalHashrate(req: Request, res: Response) { + try { + const stats = await mining.$getHistoricalHashrates(req.params.interval ?? null); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + res.json(stats); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + public async getBlock(req: Request, res: Response) { try { const result = await bitcoinApi.$getBlock(req.params.hash); diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html index e55205844..04534f176 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html @@ -1 +1,53 @@ -

hashrate-chart works!

+
+ +
+
+ +
+ +
+
+
+ + + + + + +
+
+
+ + + +
diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss index e69de29bb..c3a63e9fa 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss @@ -0,0 +1,10 @@ +.main-title { + position: relative; + color: #ffffff91; + margin-top: -13px; + font-size: 10px; + text-transform: uppercase; + font-weight: 500; + text-align: center; + padding-bottom: 3px; +} diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts index cfbb6ba31..4739c2c30 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts @@ -1,15 +1,173 @@ -import { Component, OnInit } from '@angular/core'; +import { Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core'; +import { EChartsOption } from 'echarts'; +import { Observable } from 'rxjs'; +import { map, share, startWith, switchMap, tap } from 'rxjs/operators'; +import { ApiService } from 'src/app/services/api.service'; +import { SeoService } from 'src/app/services/seo.service'; +import { formatNumber } from '@angular/common'; +import { FormBuilder, FormGroup } from '@angular/forms'; @Component({ selector: 'app-hashrate-chart', templateUrl: './hashrate-chart.component.html', - styleUrls: ['./hashrate-chart.component.scss'] + styleUrls: ['./hashrate-chart.component.scss'], + styles: [` + .loadingGraphs { + position: absolute; + top: 38%; + left: calc(50% - 15px); + z-index: 100; + } + `], }) export class HashrateChartComponent implements OnInit { + @Input() widget: boolean = false; - constructor() { } + radioGroupForm: FormGroup; - ngOnInit(): void { + chartOptions: EChartsOption = {}; + chartInitOptions = { + renderer: 'svg' + }; + + hashrateObservable$: Observable; + isLoading = true; + formatNumber = formatNumber; + + constructor( + @Inject(LOCALE_ID) public locale: string, + private seoService: SeoService, + private apiService: ApiService, + private formBuilder: FormBuilder, + ) { + this.seoService.setTitle($localize`:@@mining.hashrate:hashrate`); + this.radioGroupForm = this.formBuilder.group({ dateSpan: '1y' }); + this.radioGroupForm.controls.dateSpan.setValue('1y'); } + ngOnInit(): void { + this.hashrateObservable$ = this.radioGroupForm.get('dateSpan').valueChanges + .pipe( + startWith('1y'), + switchMap((timespan) => { + return this.apiService.getHistoricalHashrate$(timespan) + .pipe( + tap(data => { + this.prepareChartOptions(data.hashrates.map(val => [val.timestamp * 1000, val.avgHashrate])); + this.isLoading = false; + }), + map(data => { + const availableTimespanDay = ( + (new Date().getTime() / 1000) - (data.oldestIndexedBlockTimestamp / 1000) + ) / 3600 / 24; + return { + availableTimespanDay: availableTimespanDay, + data: data.hashrates + }; + }), + ); + }), + share() + ); + } + + prepareChartOptions(data) { + this.chartOptions = { + title: { + text: this.widget? '' : $localize`:@@mining.hashrate:Hashrate`, + left: 'center', + textStyle: { + color: '#FFF', + }, + }, + tooltip: { + show: true, + trigger: 'axis', + }, + axisPointer: { + type: 'line', + }, + xAxis: { + type: 'time', + splitNumber: this.isMobile() ? 5 : 10, + }, + yAxis: { + type: 'value', + axisLabel: { + formatter: (val) => { + const powerOfTen = { + exa: Math.pow(10, 18), + peta: Math.pow(10, 15), + terra: Math.pow(10, 12), + giga: Math.pow(10, 9), + mega: Math.pow(10, 6), + kilo: Math.pow(10, 3), + } + + let selectedPowerOfTen = { divider: powerOfTen.exa, unit: 'E' }; + if (val < powerOfTen.mega) { + selectedPowerOfTen = { divider: 1, unit: '' }; // no scaling + } else if (val < powerOfTen.giga) { + selectedPowerOfTen = { divider: powerOfTen.mega, unit: 'M' }; + } else if (val < powerOfTen.terra) { + selectedPowerOfTen = { divider: powerOfTen.giga, unit: 'G' }; + } else if (val < powerOfTen.peta) { + selectedPowerOfTen = { divider: powerOfTen.terra, unit: 'T' }; + } else if (val < powerOfTen.exa) { + selectedPowerOfTen = { divider: powerOfTen.peta, unit: 'P' }; + } + + const newVal = val / selectedPowerOfTen.divider; + return `${newVal} ${selectedPowerOfTen.unit}` + } + }, + splitLine: { + lineStyle: { + type: 'dotted', + color: '#ffffff66', + opacity: 0.25, + } + }, + }, + series: { + showSymbol: false, + data: data, + type: 'line', + smooth: false, + lineStyle: { + width: 3, + }, + areaStyle: {}, + }, + dataZoom: this.widget ? null : [{ + type: 'inside', + realtime: true, + zoomLock: true, + zoomOnMouseWheel: true, + moveOnMouseMove: true, + maxSpan: 100, + minSpan: 10, + }, { + showDetail: false, + show: true, + type: 'slider', + brushSelect: false, + realtime: true, + bottom: 0, + selectedDataBackground: { + lineStyle: { + color: '#fff', + opacity: 0.45, + }, + areaStyle: { + opacity: 0, + } + }, + }], + }; + } + + isMobile() { + return (window.innerWidth <= 767.98); + } } diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 9a6bbc0b8..cf0ebd414 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -156,4 +156,11 @@ export class ApiService { (interval !== undefined ? `/${interval}` : '') ); } + + getHistoricalHashrate$(interval: string | undefined): Observable { + return this.httpClient.get( + this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/hashrate` + + (interval !== undefined ? `/${interval}` : '') + ); + } } From e61df324ea55c81b9ca89cf0d6f13d6c6a878fd7 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 21 Feb 2022 12:22:20 +0900 Subject: [PATCH 04/19] Index new hashrates once every 24 hours --- backend/src/api/database-migration.ts | 4 ++ backend/src/api/mining.ts | 46 +++++++++++------ backend/src/repositories/BlocksRepository.ts | 7 +++ .../src/repositories/HashratesRepository.ts | 17 +++++++ .../difficulty-chart.component.ts | 21 ++++---- .../hashrate-chart.component.html | 49 ++++++------------- .../hashrate-chart.component.ts | 16 +++--- 7 files changed, 92 insertions(+), 68 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 154068164..098394664 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -261,6 +261,10 @@ class DatabaseMigration { } } + if (version < 7) { + queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`); + } + return queries; } diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 6f90ab357..792e93f45 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -14,7 +14,7 @@ class Mining { /** * Generate high level overview of the pool ranks and general stats */ - public async $getPoolsStats(interval: string | null) : Promise { + public async $getPoolsStats(interval: string | null): Promise { const poolsStatistics = {}; const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval); @@ -30,8 +30,8 @@ class Mining { link: poolInfo.link, blockCount: poolInfo.blockCount, rank: rank++, - emptyBlocks: 0, - } + emptyBlocks: 0 + }; for (let i = 0; i < emptyBlocks.length; ++i) { if (emptyBlocks[i].poolId === poolInfo.poolId) { poolStat.emptyBlocks++; @@ -84,32 +84,41 @@ class Mining { return { adjustments: difficultyAdjustments, oldestIndexedBlockTimestamp: oldestBlock.getTime(), - } + }; } /** * Return the historical hashrates and oldest indexed block timestamp */ - public async $getHistoricalHashrates(interval: string | null): Promise { + public async $getHistoricalHashrates(interval: string | null): Promise { const hashrates = await HashratesRepository.$get(interval); const oldestBlock = new Date(await BlocksRepository.$oldestBlockTimestamp()); return { hashrates: hashrates, oldestIndexedBlockTimestamp: oldestBlock.getTime(), - } + }; } /** - * + * Generate daily hashrate data */ - public async $generateNetworkHashrateHistory() : Promise { + public async $generateNetworkHashrateHistory(): Promise { + // We only run this once a day + const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp(); + const now = new Date().getTime() / 1000; + if (now - latestTimestamp < 86400) { + return; + } + + logger.info(`Indexing hashrates`); + if (this.hashrateIndexingStarted) { return; } this.hashrateIndexingStarted = true; - const totalIndexed = await BlocksRepository.$blockCount(null, null); + const oldestIndexedBlockHeight = await BlocksRepository.$getOldestIndexedBlockHeight(); const indexedTimestamp = (await HashratesRepository.$get(null)).map(hashrate => hashrate.timestamp); const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f @@ -128,16 +137,18 @@ class Mining { null, fromTimestamp, toTimestamp ); - let lastBlockHashrate = 0; - if (blockStats.blockCount > 0) { - lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, - blockStats.lastBlockHeight); + if (blockStats.blockCount === 0) { // We are done indexing, no blocks left + break; } - if (toTimestamp % 864000 === 0) { - const progress = Math.round((totalIndexed - blockStats.lastBlockHeight) / totalIndexed * 100); + let lastBlockHashrate = 0; + lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, + blockStats.lastBlockHeight); + + if (toTimestamp % 864000 === 0) { // Log every 10 days during initial indexing const formattedDate = new Date(fromTimestamp * 1000).toUTCString(); - logger.debug(`Counting blocks and hashrate for ${formattedDate}. Progress: ${progress}%`); + const blocksLeft = blockStats.lastBlockHeight - oldestIndexedBlockHeight; + logger.debug(`Counting blocks and hashrate for ${formattedDate}. ${blocksLeft} blocks left`); } await HashratesRepository.$saveDailyStat({ @@ -149,6 +160,9 @@ class Mining { toTimestamp -= 86400; } + await HashratesRepository.$setLatestRunTimestamp(); + this.hashrateIndexingStarted = false; + logger.info(`Hashrates indexing completed`); } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 937320a3a..9c7e9b778 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -281,6 +281,13 @@ class BlocksRepository { return rows; } + + public async $getOldestIndexedBlockHeight(): Promise { + const connection = await DB.pool.getConnection(); + const [rows]: any[] = await connection.query(`SELECT MIN(height) as minHeight FROM blocks`); + connection.release(); + return rows[0].minHeight; + } } export default new BlocksRepository(); diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 837569cd8..2a898e5bd 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -43,11 +43,28 @@ class HashratesRepository { query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`; } + query += ` ORDER by hashrate_timestamp DESC`; + const [rows]: any[] = await connection.query(query); connection.release(); return rows; } + + public async $setLatestRunTimestamp() { + const connection = await DB.pool.getConnection(); + const query = `UPDATE state SET number = ? WHERE name = 'last_hashrates_indexing'`; + await connection.query(query, [Math.round(new Date().getTime() / 1000)]); + connection.release(); + } + + public async $getLatestRunTimestamp(): Promise { + const connection = await DB.pool.getConnection(); + const query = `SELECT number FROM state WHERE name = 'last_hashrates_indexing'`; + const [rows] = await connection.query(query); + connection.release(); + return rows[0]['number']; + } } export default new HashratesRepository(); diff --git a/frontend/src/app/components/difficulty-chart/difficulty-chart.component.ts b/frontend/src/app/components/difficulty-chart/difficulty-chart.component.ts index 350e3c4be..a8865ec09 100644 --- a/frontend/src/app/components/difficulty-chart/difficulty-chart.component.ts +++ b/frontend/src/app/components/difficulty-chart/difficulty-chart.component.ts @@ -134,17 +134,18 @@ export class DifficultyChartComponent implements OnInit { } } }, - series: [ - { - data: data, - type: 'line', - smooth: false, - lineStyle: { - width: 3, - }, - areaStyle: {} + series: { + showSymbol: false, + data: data, + type: 'line', + smooth: false, + lineStyle: { + width: 2, }, - ], + areaStyle: { + opacity: 0.25 + }, + }, }; } diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html index 04534f176..cfcd15bfe 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html @@ -4,50 +4,29 @@
- -
-
+ +
+
-
- - -
diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts index 4739c2c30..8a3413db5 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts @@ -66,15 +66,15 @@ export class HashrateChartComponent implements OnInit { }; }), ); - }), - share() - ); + }), + share() + ); } prepareChartOptions(data) { this.chartOptions = { title: { - text: this.widget? '' : $localize`:@@mining.hashrate:Hashrate`, + text: this.widget ? '' : $localize`:@@mining.hashrate:Hashrate`, left: 'center', textStyle: { color: '#FFF', @@ -102,7 +102,7 @@ export class HashrateChartComponent implements OnInit { giga: Math.pow(10, 9), mega: Math.pow(10, 6), kilo: Math.pow(10, 3), - } + }; let selectedPowerOfTen = { divider: powerOfTen.exa, unit: 'E' }; if (val < powerOfTen.mega) { @@ -135,9 +135,11 @@ export class HashrateChartComponent implements OnInit { type: 'line', smooth: false, lineStyle: { - width: 3, + width: 2, + }, + areaStyle: { + opacity: 0.25 }, - areaStyle: {}, }, dataZoom: this.widget ? null : [{ type: 'inside', From 53a8d5b24639402eaebf4d8c759fe604abf7ac84 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 21 Feb 2022 12:23:32 +0900 Subject: [PATCH 05/19] Add network hashrate to mining dashboard --- .../mining-dashboard/mining-dashboard.component.html | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/frontend/src/app/components/mining-dashboard/mining-dashboard.component.html b/frontend/src/app/components/mining-dashboard/mining-dashboard.component.html index f8f52fc9a..35a26099b 100644 --- a/frontend/src/app/components/mining-dashboard/mining-dashboard.component.html +++ b/frontend/src/app/components/mining-dashboard/mining-dashboard.component.html @@ -14,6 +14,18 @@ + +
+
Hashrate (1y)
+
+
+ +
View more + »
+
+
+
+
Difficulty (1y)
From ac118141ce0feb9cba40c6e4d7bd68094d1c354f Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 21 Feb 2022 14:02:41 +0900 Subject: [PATCH 06/19] Make hashrate chart more responsive --- .../hashrate-chart.component.html | 15 ++++---- .../hashrate-chart.component.scss | 34 +++++++++++++++++++ .../hashrate-chart.component.ts | 10 +++++- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html index cfcd15bfe..263df95b2 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html @@ -1,11 +1,6 @@ -
+
-
-
- -
- -
+
+ +
+
+ +
+
diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss index c3a63e9fa..30a810ca4 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss @@ -8,3 +8,37 @@ text-align: center; padding-bottom: 3px; } + +.full-container { + width: 100%; + height: calc(100% - 100px); + @media (max-width: 992px) { + height: calc(100% - 140px); + }; + @media (max-width: 576px) { + height: calc(100% - 180px); + }; +} + +.chart { + width: 100%; + height: 100%; + padding: 1.25rem; +} + +.formRadioGroup { + margin-top: 6px; + display: flex; + flex-direction: column; + @media (min-width: 830px) { + flex-direction: row; + float: right; + margin-top: 0px; + } + .btn-sm { + font-size: 9px; + @media (min-width: 830px) { + font-size: 14px; + } + } +} diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts index 8a3413db5..24f0befd3 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts @@ -22,12 +22,16 @@ import { FormBuilder, FormGroup } from '@angular/forms'; }) export class HashrateChartComponent implements OnInit { @Input() widget: boolean = false; + @Input() right: number | string = 10; + @Input() left: number | string = 75; radioGroupForm: FormGroup; chartOptions: EChartsOption = {}; chartInitOptions = { - renderer: 'svg' + renderer: 'svg', + width: 'auto', + height: 'auto', }; hashrateObservable$: Observable; @@ -73,6 +77,10 @@ export class HashrateChartComponent implements OnInit { prepareChartOptions(data) { this.chartOptions = { + grid: { + right: this.right, + left: this.left, + }, title: { text: this.widget ? '' : $localize`:@@mining.hashrate:Hashrate`, left: 'center', From e4721e8574ebbf5890f3580556a684fc3c989ae9 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 21 Feb 2022 14:36:48 +0900 Subject: [PATCH 07/19] Improve hashrate indexing logs --- backend/src/api/mining.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 792e93f45..db73117ce 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -118,18 +118,21 @@ class Mining { } this.hashrateIndexingStarted = true; - const oldestIndexedBlockHeight = await BlocksRepository.$getOldestIndexedBlockHeight(); + const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144; const indexedTimestamp = (await HashratesRepository.$get(null)).map(hashrate => hashrate.timestamp); - + let startedAt = new Date().getTime() / 1000; const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f const lastMidnight = new Date(); lastMidnight.setUTCHours(0); lastMidnight.setUTCMinutes(0); lastMidnight.setUTCSeconds(0); lastMidnight.setUTCMilliseconds(0); let toTimestamp = Math.round(lastMidnight.getTime() / 1000); + let indexedThisRun = 0; + let totalIndexed = 0; while (toTimestamp > genesisTimestamp) { const fromTimestamp = toTimestamp - 86400; if (indexedTimestamp.includes(fromTimestamp)) { toTimestamp -= 86400; + ++totalIndexed; continue; } @@ -145,10 +148,14 @@ class Mining { lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, blockStats.lastBlockHeight); - if (toTimestamp % 864000 === 0) { // Log every 10 days during initial indexing + const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); + if (elapsedSeconds > 1) { + const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); const formattedDate = new Date(fromTimestamp * 1000).toUTCString(); - const blocksLeft = blockStats.lastBlockHeight - oldestIndexedBlockHeight; - logger.debug(`Counting blocks and hashrate for ${formattedDate}. ${blocksLeft} blocks left`); + const daysLeft = Math.round(totalDayIndexed - totalIndexed); + logger.debug(`Getting hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ~${daysLeft} days left to index`); + startedAt = new Date().getTime() / 1000; + indexedThisRun = 0; } await HashratesRepository.$saveDailyStat({ @@ -158,6 +165,8 @@ class Mining { }); toTimestamp -= 86400; + ++indexedThisRun; + ++totalIndexed; } await HashratesRepository.$setLatestRunTimestamp(); From e5907159b8db04c0e811a3193e2d9aafcfbbec12 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 21 Feb 2022 15:55:27 +0900 Subject: [PATCH 08/19] Refactor power of ten conversion into one wrapper --- frontend/src/app/bitcoin.utils.ts | 29 +++++++++++++++++++ .../difficulty-chart.component.ts | 10 ++----- .../hashrate-chart.component.ts | 24 ++------------- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/frontend/src/app/bitcoin.utils.ts b/frontend/src/app/bitcoin.utils.ts index ff2d7a885..72fde7471 100644 --- a/frontend/src/app/bitcoin.utils.ts +++ b/frontend/src/app/bitcoin.utils.ts @@ -130,3 +130,32 @@ export const formatNumber = (s, precision = null) => { // Utilities for segwitFeeGains const witnessSize = (vin: Vin) => vin.witness.reduce((S, w) => S + (w.length / 2), 0); const scriptSigSize = (vin: Vin) => vin.scriptsig ? vin.scriptsig.length / 2 : 0; + +// Power of ten wrapper +export function selectPowerOfTen(val: number) { + const powerOfTen = { + exa: Math.pow(10, 18), + peta: Math.pow(10, 15), + terra: Math.pow(10, 12), + giga: Math.pow(10, 9), + mega: Math.pow(10, 6), + kilo: Math.pow(10, 3), + }; + + let selectedPowerOfTen; + if (val < powerOfTen.mega) { + selectedPowerOfTen = { divider: 1, unit: '' }; // no scaling + } else if (val < powerOfTen.giga) { + selectedPowerOfTen = { divider: powerOfTen.mega, unit: 'M' }; + } else if (val < powerOfTen.terra) { + selectedPowerOfTen = { divider: powerOfTen.giga, unit: 'G' }; + } else if (val < powerOfTen.peta) { + selectedPowerOfTen = { divider: powerOfTen.terra, unit: 'T' }; + } else if (val < powerOfTen.exa) { + selectedPowerOfTen = { divider: powerOfTen.peta, unit: 'P' }; + } else { + selectedPowerOfTen = { divider: powerOfTen.exa, unit: 'E' }; + } + + return selectedPowerOfTen; +} \ No newline at end of file diff --git a/frontend/src/app/components/difficulty-chart/difficulty-chart.component.ts b/frontend/src/app/components/difficulty-chart/difficulty-chart.component.ts index a8865ec09..6ce15599c 100644 --- a/frontend/src/app/components/difficulty-chart/difficulty-chart.component.ts +++ b/frontend/src/app/components/difficulty-chart/difficulty-chart.component.ts @@ -6,6 +6,7 @@ import { ApiService } from 'src/app/services/api.service'; import { SeoService } from 'src/app/services/seo.service'; import { formatNumber } from '@angular/common'; import { FormBuilder, FormGroup } from '@angular/forms'; +import { selectPowerOfTen } from 'src/app/bitcoin.utils'; @Component({ selector: 'app-difficulty-chart', @@ -70,15 +71,8 @@ export class DifficultyChartComponent implements OnInit { const tableData = []; for (let i = 0; i < data.adjustments.length - 1; ++i) { + const selectedPowerOfTen: any = selectPowerOfTen(data.adjustments[i].difficulty); const change = (data.adjustments[i].difficulty / data.adjustments[i + 1].difficulty - 1) * 100; - let selectedPowerOfTen = { divider: powerOfTen.terra, unit: 'T' }; - if (data.adjustments[i].difficulty < powerOfTen.mega) { - selectedPowerOfTen = { divider: 1, unit: '' }; // no scaling - } else if (data.adjustments[i].difficulty < powerOfTen.giga) { - selectedPowerOfTen = { divider: powerOfTen.mega, unit: 'M' }; - } else if (data.adjustments[i].difficulty < powerOfTen.terra) { - selectedPowerOfTen = { divider: powerOfTen.giga, unit: 'G' }; - } tableData.push(Object.assign(data.adjustments[i], { change: change, diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts index 24f0befd3..0f5a6a98e 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts @@ -6,6 +6,7 @@ import { ApiService } from 'src/app/services/api.service'; import { SeoService } from 'src/app/services/seo.service'; import { formatNumber } from '@angular/common'; import { FormBuilder, FormGroup } from '@angular/forms'; +import { selectPowerOfTen } from 'src/app/bitcoin.utils'; @Component({ selector: 'app-hashrate-chart', @@ -103,28 +104,7 @@ export class HashrateChartComponent implements OnInit { type: 'value', axisLabel: { formatter: (val) => { - const powerOfTen = { - exa: Math.pow(10, 18), - peta: Math.pow(10, 15), - terra: Math.pow(10, 12), - giga: Math.pow(10, 9), - mega: Math.pow(10, 6), - kilo: Math.pow(10, 3), - }; - - let selectedPowerOfTen = { divider: powerOfTen.exa, unit: 'E' }; - if (val < powerOfTen.mega) { - selectedPowerOfTen = { divider: 1, unit: '' }; // no scaling - } else if (val < powerOfTen.giga) { - selectedPowerOfTen = { divider: powerOfTen.mega, unit: 'M' }; - } else if (val < powerOfTen.terra) { - selectedPowerOfTen = { divider: powerOfTen.giga, unit: 'G' }; - } else if (val < powerOfTen.peta) { - selectedPowerOfTen = { divider: powerOfTen.terra, unit: 'T' }; - } else if (val < powerOfTen.exa) { - selectedPowerOfTen = { divider: powerOfTen.peta, unit: 'P' }; - } - + const selectedPowerOfTen: any = selectPowerOfTen(val); const newVal = val / selectedPowerOfTen.divider; return `${newVal} ${selectedPowerOfTen.unit}` } From bb1c5d0b31948710dc69737d26c448134a4ac882 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 21 Feb 2022 16:38:18 +0900 Subject: [PATCH 09/19] Add `--reindex` command line parameter to force full re-indexing --- backend/src/api/database-migration.ts | 23 +++++++++++++++++++++++ backend/src/index.ts | 6 ++++++ 2 files changed, 29 insertions(+) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 098394664..3e9762316 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -419,6 +419,29 @@ class DatabaseMigration { FOREIGN KEY (pool_id) REFERENCES pools (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + + public async $truncateIndexedData(tables: string[]) { + const allowedTables = ['blocks', 'hashrates']; + + const connection = await DB.pool.getConnection(); + try { + for (const table of tables) { + if (!allowedTables.includes(table)) { + logger.info(`Table ${table} cannot to be re-indexed (not allowed)`); + continue; + }; + + await this.$executeQuery(connection, `TRUNCATE ${table}`, true); + if (table === 'hashrates') { + await this.$executeQuery(connection, 'UPDATE state set number = 0 where name = "last_hashrates_indexing"', true); + } + logger.info(`Table ${table} has been truncated`); + } + } catch (e) { + logger.warn(`Unable to erase indexed data`); + } + connection.release(); + } } export default new DatabaseMigration(); diff --git a/backend/src/index.ts b/backend/src/index.ts index 4fe66bc72..64ec25382 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -89,6 +89,12 @@ class Server { if (config.DATABASE.ENABLED) { await checkDbConnection(); try { + if (process.env.npm_config_reindex != undefined) { // Re-index requests + const tables = process.env.npm_config_reindex.split(','); + logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds from now (using '--reindex') ...`); + await Common.sleep(5000); + await databaseMigration.$truncateIndexedData(tables); + } await databaseMigration.$initializeOrMigrateDatabase(); await poolsParser.migratePoolsJson(); } catch (e) { From 537e50c682b464ae69a451ff64c4e67fdbe25085 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 21 Feb 2022 16:54:43 +0900 Subject: [PATCH 10/19] Reduce log spam during hashrate indexing --- backend/src/api/mining.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index db73117ce..2411420cb 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -111,13 +111,13 @@ class Mining { return; } - logger.info(`Indexing hashrates`); - if (this.hashrateIndexingStarted) { return; } this.hashrateIndexingStarted = true; + logger.info(`Indexing hashrates`); + const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144; const indexedTimestamp = (await HashratesRepository.$get(null)).map(hashrate => hashrate.timestamp); let startedAt = new Date().getTime() / 1000; @@ -149,7 +149,7 @@ class Mining { blockStats.lastBlockHeight); const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - if (elapsedSeconds > 1) { + if (elapsedSeconds > 10) { const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); const formattedDate = new Date(fromTimestamp * 1000).toUTCString(); const daysLeft = Math.round(totalDayIndexed - totalIndexed); From 649ad2e859e9a6ebdd8b3fcb292112aaa91c2fb2 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 21 Feb 2022 17:34:07 +0900 Subject: [PATCH 11/19] Hashrates indexing waits for blocks indexing - Batch hashrates I/O ops --- backend/src/api/blocks.ts | 3 +++ backend/src/api/mining.ts | 18 ++++++++++----- backend/src/index.ts | 6 ++--- .../src/repositories/HashratesRepository.ts | 22 +++++++++---------- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 483fd3f3e..de461e095 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -20,6 +20,7 @@ class Blocks { private previousDifficultyRetarget = 0; private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; private blockIndexingStarted = false; + public blockIndexingCompleted = false; constructor() { } @@ -240,6 +241,8 @@ class Blocks { logger.err('An error occured in $generateBlockDatabase(). Skipping block indexing. ' + e); console.log(e); } + + this.blockIndexingCompleted = true; } public async $updateBlocks() { diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 2411420cb..1d5142080 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -4,6 +4,7 @@ import PoolsRepository from '../repositories/PoolsRepository'; import HashratesRepository from '../repositories/HashratesRepository'; import bitcoinClient from './bitcoin/bitcoin-client'; import logger from '../logger'; +import blocks from './blocks'; class Mining { hashrateIndexingStarted = false; @@ -111,7 +112,7 @@ class Mining { return; } - if (this.hashrateIndexingStarted) { + if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) { return; } this.hashrateIndexingStarted = true; @@ -128,6 +129,8 @@ class Mining { let indexedThisRun = 0; let totalIndexed = 0; + const hashrates: any[] = []; + while (toTimestamp > genesisTimestamp) { const fromTimestamp = toTimestamp - 86400; if (indexedTimestamp.includes(fromTimestamp)) { @@ -137,9 +140,7 @@ class Mining { } const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp( - null, fromTimestamp, toTimestamp - ); - + null, fromTimestamp, toTimestamp); if (blockStats.blockCount === 0) { // We are done indexing, no blocks left break; } @@ -158,23 +159,28 @@ class Mining { indexedThisRun = 0; } - await HashratesRepository.$saveDailyStat({ + hashrates.push({ hashrateTimestamp: fromTimestamp, avgHashrate: lastBlockHashrate, poolId: null, }); + if (hashrates.length > 100) { + await HashratesRepository.$saveHashrates(hashrates); + hashrates.length = 0; + } + toTimestamp -= 86400; ++indexedThisRun; ++totalIndexed; } + await HashratesRepository.$saveHashrates(hashrates); await HashratesRepository.$setLatestRunTimestamp(); this.hashrateIndexingStarted = false; logger.info(`Hashrates indexing completed`); } - } export default new Mining(); diff --git a/backend/src/index.ts b/backend/src/index.ts index 64ec25382..d051766fa 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -164,12 +164,12 @@ class Server { } } - async runIndexingWhenReady() { + runIndexingWhenReady() { if (!Common.indexingEnabled() || mempool.hasPriority()) { return; } - await blocks.$generateBlockDatabase(); - await mining.$generateNetworkHashrateHistory(); + blocks.$generateBlockDatabase(); + mining.$generateNetworkHashrateHistory(); } setUpWebsocketHandling() { diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 2a898e5bd..0e8f1477e 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -6,21 +6,19 @@ class HashratesRepository { /** * Save indexed block data in the database */ - public async $saveDailyStat(dailyStat: any) { + public async $saveHashrates(hashrates: any) { + let query = `INSERT INTO + hashrates(hashrate_timestamp, avg_hashrate, pool_id) VALUES`; + + for (const hashrate of hashrates) { + query += ` (FROM_UNIXTIME(${hashrate.hashrateTimestamp}), ${hashrate.avgHashrate}, ${hashrate.poolId}),`; + } + query = query.slice(0, -1); + const connection = await DB.pool.getConnection(); - try { - const query = `INSERT INTO - hashrates(hashrate_timestamp, avg_hashrate, pool_id) - VALUE (FROM_UNIXTIME(?), ?, ?)`; - - const params: any[] = [ - dailyStat.hashrateTimestamp, dailyStat.avgHashrate, - dailyStat.poolId - ]; - // logger.debug(query); - await connection.query(query, params); + await connection.query(query); } catch (e: any) { logger.err('$saveHashrateInDatabase() error' + (e instanceof Error ? e.message : e)); } From 6e62c628553e805f0bd3a1063d401d125ada589b Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 21 Feb 2022 18:01:09 +0900 Subject: [PATCH 12/19] Add /api/v1/mining/hashrate/* apis to the cache warmer --- production/nginx-cache-warmer | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer index e3ecf6c91..8c679980c 100755 --- a/production/nginx-cache-warmer +++ b/production/nginx-cache-warmer @@ -21,6 +21,12 @@ do for url in / \ '/api/v1/mining/pools/2y' \ '/api/v1/mining/pools/3y' \ '/api/v1/mining/pools/all' \ + '/api/v1/mining/hashrate/3m' \ + '/api/v1/mining/hashrate/6m' \ + '/api/v1/mining/hashrate/1y' \ + '/api/v1/mining/hashrate/2y' \ + '/api/v1/mining/hashrate/3y' \ + '/api/v1/mining/hashrate/all' \ do curl -s "https://${hostname}${url}" >/dev/null From 413cf3ccaafa68c7c2f40250d051b6ac5feaea87 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 21 Feb 2022 18:19:03 +0900 Subject: [PATCH 13/19] Fix 'active' menu when using mining dashboard --- .../src/app/components/master-page/master-page.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/master-page/master-page.component.html b/frontend/src/app/components/master-page/master-page.component.html index bab0f42e9..28db64bbd 100644 --- a/frontend/src/app/components/master-page/master-page.component.html +++ b/frontend/src/app/components/master-page/master-page.component.html @@ -31,7 +31,7 @@ -