From 33897b029f414b8d89ba3c3c81060f035244820f Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 12 Mar 2022 14:47:33 +0100 Subject: [PATCH 1/5] Set db connection to UTC - Fix hashrate indexing --- backend/src/api/database-migration.ts | 25 ++++++++++------- backend/src/api/liquid/elements-parser.ts | 8 +++--- backend/src/api/mining.ts | 13 +++++---- backend/src/api/pools-parser.ts | 4 +-- backend/src/api/statistics.ts | 24 ++++++++--------- backend/src/database.ts | 17 +++++++++--- backend/src/index.ts | 6 ++--- backend/src/repositories/BlocksRepository.ts | 20 +++++++------- .../src/repositories/HashratesRepository.ts | 27 ++++++++++--------- backend/src/repositories/PoolsRepository.ts | 10 +++---- 10 files changed, 86 insertions(+), 68 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index bdad06961..c9c1da8e8 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,7 +6,7 @@ import logger from '../logger'; const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 13; + private static currentVersion = 14; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -77,7 +77,7 @@ class DatabaseMigration { await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion); const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { await this.$executeQuery(connection, this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs')); await this.$executeQuery(connection, this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics')); @@ -168,6 +168,13 @@ class DatabaseMigration { await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); } + if (databaseSchemaVersion < 14 && isBitcoin === true) { + logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`); + await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index + await this.$executeQuery(connection, 'ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`'); + await this.$executeQuery(connection, 'ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"'); + } + connection.release(); } catch (e) { connection.release(); @@ -187,7 +194,7 @@ class DatabaseMigration { return; } - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { // We don't use "CREATE INDEX IF NOT EXISTS" because it is not supported on old mariadb version 5.X @@ -225,7 +232,7 @@ class DatabaseMigration { * Check if 'table' exists in the database */ private async $checkIfTableExists(table: string): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${config.DATABASE.DATABASE}' AND TABLE_NAME = '${table}'`; const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -236,7 +243,7 @@ class DatabaseMigration { * Get current database version */ private async $getSchemaVersionFromDatabase(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT number FROM state WHERE name = 'schema_version';`; const [rows] = await this.$executeQuery(connection, query, true); connection.release(); @@ -247,7 +254,7 @@ class DatabaseMigration { * Create the `state` table */ private async $createMigrationStateTable(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const query = `CREATE TABLE IF NOT EXISTS state ( @@ -279,7 +286,7 @@ class DatabaseMigration { } transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery()); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { await this.$executeQuery(connection, 'START TRANSACTION;'); for (const query of transactionQueries) { @@ -330,7 +337,7 @@ class DatabaseMigration { * Print current database version */ private async $printDatabaseVersion() { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await this.$executeQuery(connection, 'SELECT VERSION() as version;', true); logger.debug(`MIGRATIONS: Database engine version '${rows[0].version}'`); @@ -474,7 +481,7 @@ class DatabaseMigration { public async $truncateIndexedData(tables: string[]) { const allowedTables = ['blocks', 'hashrates']; - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { for (const table of tables) { if (!allowedTables.includes(table)) { diff --git a/backend/src/api/liquid/elements-parser.ts b/backend/src/api/liquid/elements-parser.ts index a2d4e1546..24c7ab949 100644 --- a/backend/src/api/liquid/elements-parser.ts +++ b/backend/src/api/liquid/elements-parser.ts @@ -33,7 +33,7 @@ class ElementsParser { } public async $getPegDataByMonth(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`; const [rows] = await connection.query(query); connection.release(); @@ -79,7 +79,7 @@ class ElementsParser { protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string, txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `INSERT INTO elements_pegs( block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`; @@ -93,7 +93,7 @@ class ElementsParser { } protected async $getLatestBlockHeightFromDatabase(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT number FROM state WHERE name = 'last_elements_block'`; const [rows] = await connection.query(query); connection.release(); @@ -101,7 +101,7 @@ class ElementsParser { } protected async $saveLatestBlockToDatabase(blockHeight: number) { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`; await connection.query(query, [blockHeight]); connection.release(); diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index afcc89220..512388b36 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -80,8 +80,8 @@ class Mining { // We only run this once a week const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_weekly_hashrates_indexing'); - const now = new Date().getTime() / 1000; - if (now - latestTimestamp < 604800) { + const now = new Date(); + if ((now.getTime() / 1000) - latestTimestamp < 604800) { return; } @@ -94,7 +94,6 @@ class Mining { const hashrates: any[] = []; const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f - const now = new Date(); const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7)); const lastMondayMidnight = this.getDateMidnight(lastMonday); let toTimestamp = Math.round((lastMondayMidnight.getTime() - 604800) / 1000); @@ -108,7 +107,7 @@ class Mining { const fromTimestamp = toTimestamp - 604800; // Skip already indexed weeks - if (indexedTimestamp.includes(toTimestamp + 1)) { + if (indexedTimestamp.includes(toTimestamp)) { toTimestamp -= 604800; ++totalIndexed; continue; @@ -133,7 +132,7 @@ class Mining { for (const pool of pools) { hashrates.push({ - hashrateTimestamp: toTimestamp + 1, + hashrateTimestamp: toTimestamp, avgHashrate: pool['hashrate'], poolId: pool.poolId, share: pool['share'], @@ -202,7 +201,7 @@ class Mining { const fromTimestamp = toTimestamp - 86400; // Skip already indexed weeks - if (indexedTimestamp.includes(fromTimestamp)) { + if (indexedTimestamp.includes(toTimestamp)) { toTimestamp -= 86400; ++totalIndexed; continue; @@ -220,7 +219,7 @@ class Mining { hashrates.push({ hashrateTimestamp: toTimestamp, avgHashrate: lastBlockHashrate, - poolId: null, + poolId: 0, share: 1, type: 'daily', }); diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 194ce0dd9..ff70c3cb9 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -66,7 +66,7 @@ class PoolsParser { logger.debug(`Found ${poolNames.length} unique mining pools`); // Get existing pools from the db - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); let existingPools; try { [existingPools] = await connection.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); @@ -152,7 +152,7 @@ class PoolsParser { * Manually add the 'unknown pool' */ private async insertUnknownPool() { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows]: any[] = await connection.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 }); if (rows.length === 0) { diff --git a/backend/src/api/statistics.ts b/backend/src/api/statistics.ts index 886ad78ba..3d99adcb7 100644 --- a/backend/src/api/statistics.ts +++ b/backend/src/api/statistics.ts @@ -155,7 +155,7 @@ class Statistics { } private async $createZeroedStatistic(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const query = `INSERT INTO statistics( added, @@ -216,7 +216,7 @@ class Statistics { } private async $create(statistics: Statistic): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const query = `INSERT INTO statistics( added, @@ -421,7 +421,7 @@ class Statistics { private async $get(id: number): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE id = ?`; const [rows] = await connection.query(query, [id]); connection.release(); @@ -435,7 +435,7 @@ class Statistics { public async $list2H(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 120`; const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -448,7 +448,7 @@ class Statistics { public async $list24H(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 1440`; const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -461,7 +461,7 @@ class Statistics { public async $list1W(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDaysAvg(300, '1 WEEK'); // 5m interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -474,7 +474,7 @@ class Statistics { public async $list1M(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDaysAvg(1800, '1 MONTH'); // 30m interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -487,7 +487,7 @@ class Statistics { public async $list3M(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDaysAvg(7200, '3 MONTH'); // 2h interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -500,7 +500,7 @@ class Statistics { public async $list6M(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDaysAvg(10800, '6 MONTH'); // 3h interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -513,7 +513,7 @@ class Statistics { public async $list1Y(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDays(28800, '1 YEAR'); // 8h interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -526,7 +526,7 @@ class Statistics { public async $list2Y(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDays(28800, "2 YEAR"); // 8h interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -539,7 +539,7 @@ class Statistics { public async $list3Y(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDays(43200, "3 YEAR"); // 12h interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); diff --git a/backend/src/database.ts b/backend/src/database.ts index 9f2655016..596ce2364 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -1,5 +1,5 @@ import config from './config'; -import { createPool } from 'mysql2/promise'; +import { createPool, PoolConnection } from 'mysql2/promise'; import logger from './logger'; export class DB { @@ -11,13 +11,24 @@ export class DB { password: config.DATABASE.PASSWORD, connectionLimit: 10, supportBigNumbers: true, - timezone: '+00:00', }); + + static connectionsReady: number[] = []; + + static async getConnection() { + const connection: PoolConnection = await DB.pool.getConnection(); + const connectionId = connection['connection'].connectionId; + if (!DB.connectionsReady.includes(connectionId)) { + await connection.query(`SET time_zone='+00:00';`); + this.connectionsReady.push(connectionId); + } + return connection; + } } export async function checkDbConnection() { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); logger.info('Database connection established.'); connection.release(); } catch (e) { diff --git a/backend/src/index.ts b/backend/src/index.ts index 0a6080b16..d4b55e078 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -5,7 +5,7 @@ import * as WebSocket from 'ws'; import * as cluster from 'cluster'; import axios from 'axios'; -import { checkDbConnection } from './database'; +import { checkDbConnection, DB } from './database'; import config from './config'; import routes from './routes'; import blocks from './api/blocks'; @@ -180,8 +180,8 @@ class Server { try { blocks.$generateBlockDatabase(); - mining.$generateNetworkHashrateHistory(); - mining.$generatePoolHashrateHistory(); + await mining.$generateNetworkHashrateHistory(); + await mining.$generatePoolHashrateHistory(); } catch (e) { logger.err(`Unable to run indexing right now, trying again later. ` + e); } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 041086f73..0af5e2252 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -8,7 +8,7 @@ class BlocksRepository { * Save indexed block data in the database */ public async $saveBlockInDatabase(block: BlockExtended) { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const query = `INSERT INTO blocks( @@ -70,7 +70,7 @@ class BlocksRepository { return []; } - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows]: any[] = await connection.query(` SELECT height @@ -116,7 +116,7 @@ class BlocksRepository { query += ` GROUP by pools.id`; - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query, params); connection.release(); @@ -154,7 +154,7 @@ class BlocksRepository { } // logger.debug(query); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query, params); connection.release(); @@ -194,7 +194,7 @@ class BlocksRepository { query += ` blockTimestamp BETWEEN FROM_UNIXTIME('${from}') AND FROM_UNIXTIME('${to}')`; // logger.debug(query); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query, params); connection.release(); @@ -217,7 +217,7 @@ class BlocksRepository { LIMIT 1;`; // logger.debug(query); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows]: any[] = await connection.query(query); connection.release(); @@ -253,7 +253,7 @@ class BlocksRepository { LIMIT 10`; // logger.debug(query); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query, params); connection.release(); @@ -274,7 +274,7 @@ class BlocksRepository { * Get one block by height */ public async $getBlockByHeight(height: number): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows]: any[] = await connection.query(` SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, @@ -305,7 +305,7 @@ class BlocksRepository { public async $getBlocksDifficulty(interval: string | null): Promise { interval = Common.getSqlInterval(interval); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); // :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162 // Basically, using temporary user defined fields, we are able to extract all @@ -356,7 +356,7 @@ class BlocksRepository { } public async $getOldestIndexedBlockHeight(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows]: any[] = await connection.query(`SELECT MIN(height) as minHeight FROM blocks`); connection.release(); diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 749d3cb57..5237e6cb7 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -20,9 +20,9 @@ class HashratesRepository { } query = query.slice(0, -1); - const connection = await DB.pool.getConnection(); + let connection; try { - // logger.debug(query); + connection = await DB.getConnection(); await connection.query(query); connection.release(); } catch (e: any) { @@ -35,18 +35,16 @@ class HashratesRepository { public async $getNetworkDailyHashrate(interval: string | null): Promise { interval = Common.getSqlInterval(interval); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate FROM hashrates`; if (interval) { query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() - AND hashrates.type = 'daily' - AND pool_id IS NULL`; + AND hashrates.type = 'daily'`; } else { - query += ` WHERE hashrates.type = 'daily' - AND pool_id IS NULL`; + query += ` WHERE hashrates.type = 'daily'`; } query += ` ORDER by hashrate_timestamp`; @@ -64,9 +62,12 @@ class HashratesRepository { } public async $getWeeklyHashrateTimestamps(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); - const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp FROM hashrates where type = 'weekly' GROUP BY hashrate_timestamp`; + const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp + FROM hashrates + WHERE type = 'weekly' + GROUP BY hashrate_timestamp`; try { const [rows]: any[] = await connection.query(query); @@ -86,7 +87,7 @@ class HashratesRepository { public async $getPoolsWeeklyHashrate(interval: string | null): Promise { interval = Common.getSqlInterval(interval); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const topPoolsId = (await PoolsRepository.$getPoolsInfo('1w')).map((pool) => pool.poolId); let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName @@ -120,7 +121,7 @@ class HashratesRepository { * Returns a pool hashrate history */ public async $getPoolWeeklyHashrate(poolId: number): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); // Find hashrate boundaries let query = `SELECT MIN(hashrate_timestamp) as firstTimestamp, MAX(hashrate_timestamp) as lastTimestamp @@ -163,7 +164,7 @@ class HashratesRepository { } public async $setLatestRunTimestamp(key: string, val: any = null) { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `UPDATE state SET number = ? WHERE name = ?`; try { @@ -175,7 +176,7 @@ class HashratesRepository { } public async $getLatestRunTimestamp(key: string): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT number FROM state WHERE name = ?`; try { diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index 3f904888d..4c3fd67ce 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -8,7 +8,7 @@ class PoolsRepository { * Get all pools tagging info */ public async $getPools(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const [rows] = await connection.query('SELECT id, name, addresses, regexes FROM pools;'); connection.release(); return rows; @@ -18,7 +18,7 @@ class PoolsRepository { * Get unknown pool tagging info */ public async $getUnknownPool(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const [rows] = await connection.query('SELECT id, name FROM pools where name = "Unknown"'); connection.release(); return rows[0]; @@ -42,7 +42,7 @@ class PoolsRepository { ORDER BY COUNT(height) DESC`; // logger.debug(query); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query); connection.release(); @@ -64,7 +64,7 @@ class PoolsRepository { LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?) GROUP BY pools.id`; - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query, [from, to]); connection.release(); @@ -87,7 +87,7 @@ class PoolsRepository { WHERE pools.id = ?`; // logger.debug(query); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query, [poolId]); connection.release(); From ab486bfe6e1c124bdc965b1e080f13f7b280bf24 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 13 Mar 2022 11:37:56 +0100 Subject: [PATCH 2/5] Use correct url for blocks-extras API - Fix amountShortner pipe --- ...ifficulty-adjustments-table.component.html | 2 +- ...difficulty-adjustments-table.components.ts | 2 +- frontend/src/app/services/api.service.ts | 2 +- .../app/shared/pipes/amount-shortener.pipe.ts | 45 +++++++++++++------ 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html b/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html index fd881016a..51872c932 100644 --- a/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html +++ b/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html @@ -17,7 +17,7 @@ {{ diffChange.difficultyShorten }} - {{ diffChange.change >= 0 ? '+' : '' }}{{ formatNumber(diffChange.change, locale, '1.2-2') }}% + {{ diffChange.change >= 0 ? '+' : '' }}{{ diffChange.change | amountShortener }}% diff --git a/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts b/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts index 24c44fe05..5e8b3ded7 100644 --- a/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts +++ b/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts @@ -43,7 +43,7 @@ export class DifficultyAdjustmentsTable implements OnInit { const change = (data.difficulty[i].difficulty / data.difficulty[i - 1].difficulty - 1) * 100; tableData.push(Object.assign(data.difficulty[i], { - change: change, + change: Math.round(change * 100) / 100, difficultyShorten: formatNumber( data.difficulty[i].difficulty / selectedPowerOfTen.divider, this.locale, '1.2-2') + selectedPowerOfTen.unit diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 142f26807..da228a833 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -153,7 +153,7 @@ export class ApiService { getBlocks$(from: number): Observable { return this.httpClient.get( - this.apiBasePath + this.apiBasePath + `/api/v1/blocks-extras` + + this.apiBaseUrl + this.apiBasePath + `/api/v1/blocks-extras` + (from !== undefined ? `/${from}` : ``) ); } diff --git a/frontend/src/app/shared/pipes/amount-shortener.pipe.ts b/frontend/src/app/shared/pipes/amount-shortener.pipe.ts index 49b452cd9..5c58b7513 100644 --- a/frontend/src/app/shared/pipes/amount-shortener.pipe.ts +++ b/frontend/src/app/shared/pipes/amount-shortener.pipe.ts @@ -1,22 +1,39 @@ import { Pipe, PipeTransform } from '@angular/core'; +// https://medium.com/@thunderroid/angular-short-number-suffix-pipe-1k-2m-3b-dded4af82fb4 + @Pipe({ name: 'amountShortener' }) export class AmountShortenerPipe implements PipeTransform { - transform(num: number, ...args: number[]): unknown { - const digits = args[0] || 1; - const lookup = [ - { value: 1, symbol: '' }, - { value: 1e3, symbol: 'k' }, - { value: 1e6, symbol: 'M' }, - { value: 1e9, symbol: 'G' }, - { value: 1e12, symbol: 'T' }, - { value: 1e15, symbol: 'P' }, - { value: 1e18, symbol: 'E' } + transform(number: number, args?: any): any { + if (isNaN(number)) return null; // will only work value is a number + if (number === null) return null; + if (number === 0) return null; + let abs = Math.abs(number); + const rounder = Math.pow(10, 1); + const isNegative = number < 0; // will also work for Negetive numbers + let key = ''; + + const powers = [ + { key: 'E', value: 10e18 }, + { key: 'P', value: 10e15 }, + { key: 'T', value: 10e12 }, + { key: 'B', value: 10e9 }, + { key: 'M', value: 10e6 }, + { key: 'K', value: 1000 } ]; - const rx = /\.0+$|(\.[0-9]*[1-9])0+$/; - var item = lookup.slice().reverse().find((item) => num >= item.value); - return item ? (num / item.value).toFixed(digits).replace(rx, '$1') + item.symbol : '0'; + + for (let i = 0; i < powers.length; i++) { + let reduced = abs / powers[i].value; + reduced = Math.round(reduced * rounder) / rounder; + if (reduced >= 1) { + abs = reduced; + key = powers[i].key; + break; + } + } + + return (isNegative ? '-' : '') + abs + key; } -} +} \ No newline at end of file From bec3f214b55b315338021b9f04c9f249b8dfe429 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 13 Mar 2022 11:38:33 +0100 Subject: [PATCH 3/5] Make sure to set avg_hashrate field to double unsigned --- backend/src/api/database-migration.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index c9c1da8e8..f68f7f17d 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,7 +6,7 @@ import logger from '../logger'; const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 14; + private static currentVersion = 15; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -175,6 +175,12 @@ class DatabaseMigration { await this.$executeQuery(connection, 'ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"'); } + if (databaseSchemaVersion < 15 && isBitcoin === true) { + logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`); + await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index + await this.$executeQuery(connection, 'ALTER TABLE `hashrates` MODIFY `avg_hashrate` DOUBLE UNSIGNED NOT NULL DEFAULT "0"'); + } + connection.release(); } catch (e) { connection.release(); From 0730053d5d3c1ae66e5b40e888450248123a96eb Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 13 Mar 2022 11:38:45 +0100 Subject: [PATCH 4/5] Use bitcoin RPC getblock because esplora returns int for difficulty - Fix some css in mining dashboard --- backend/src/api/bitcoin/bitcoin-api.ts | 4 +- backend/src/api/blocks.ts | 37 +++++----- .../mining-dashboard.component.html | 62 ++++++++-------- .../mining-dashboard.component.scss | 71 ++++++++++--------- .../mining-dashboard.component.ts | 4 +- 5 files changed, 94 insertions(+), 84 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 97a428c23..8d66f82ef 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -70,7 +70,7 @@ class BitcoinApi implements AbstractBitcoinApi { } return this.bitcoindClient.getBlock(hash) - .then((block: IBitcoinApi.Block) => this.convertBlock(block)); + .then((block: IBitcoinApi.Block) => BitcoinApi.convertBlock(block)); } $getAddress(address: string): Promise { @@ -186,7 +186,7 @@ class BitcoinApi implements AbstractBitcoinApi { return esploraTransaction; } - private convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block { + static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block { return { id: block.hash, height: block.height, diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 7a9589348..8f066b5a4 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -11,6 +11,7 @@ import { IEsploraApi } from './bitcoin/esplora-api.interface'; import poolsRepository from '../repositories/PoolsRepository'; import blocksRepository from '../repositories/BlocksRepository'; import loadingIndicators from './loading-indicators'; +import BitcoinApi from './bitcoin/bitcoin-api'; class Blocks { private blocks: BlockExtended[] = []; @@ -103,8 +104,8 @@ class Blocks { * @param transactions * @returns BlockExtended */ - private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { - const blockExtended: BlockExtended = Object.assign({extras: {}}, block); + private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { + const blockExtended: BlockExtended = Object.assign({ extras: {} }, block); blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); @@ -112,19 +113,19 @@ class Blocks { blockExtended.extras.coinbaseRaw = coinbaseRaw.hex; if (block.height === 0) { - blockExtended.extras.medianFee = 0; // 50th percentiles - blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; - blockExtended.extras.totalFees = 0; - blockExtended.extras.avgFee = 0; - blockExtended.extras.avgFeeRate = 0; - } else { + blockExtended.extras.medianFee = 0; // 50th percentiles + blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; + blockExtended.extras.totalFees = 0; + blockExtended.extras.avgFee = 0; + blockExtended.extras.avgFeeRate = 0; + } else { const stats = await bitcoinClient.getBlockStats(block.id); blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles - blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); - blockExtended.extras.totalFees = stats.totalfee; - blockExtended.extras.avgFee = stats.avgfee; - blockExtended.extras.avgFeeRate = stats.avgfeerate; - } + blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); + blockExtended.extras.totalFees = stats.totalfee; + blockExtended.extras.avgFee = stats.avgfee; + blockExtended.extras.avgFeeRate = stats.avgfeerate; + } if (Common.indexingEnabled()) { let pool: PoolTag; @@ -239,7 +240,7 @@ class Blocks { indexedThisRun = 0; } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); - const block = await bitcoinApi.$getBlock(blockHash); + const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true); const blockExtended = await this.$getBlockExtended(block, transactions); await blocksRepository.$saveBlockInDatabase(blockExtended); @@ -276,7 +277,7 @@ class Blocks { if (blockchainInfo.blocks === blockchainInfo.headers) { const heightDiff = blockHeightTip % 2016; const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); - const block = await bitcoinApi.$getBlock(blockHash); + const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); this.lastDifficultyAdjustmentTime = block.timestamp; this.currentDifficulty = block.difficulty; @@ -300,7 +301,7 @@ class Blocks { } const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight); - const block = await bitcoinApi.$getBlock(blockHash); + const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, false); const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions); @@ -332,14 +333,14 @@ class Blocks { /** * Index a block if it's missing from the database. Returns the block after indexing */ - public async $indexBlock(height: number): Promise { + public async $indexBlock(height: number): Promise { const dbBlock = await blocksRepository.$getBlockByHeight(height); if (dbBlock != null) { return this.prepareBlock(dbBlock); } const blockHash = await bitcoinApi.$getBlockHash(height); - const block = await bitcoinApi.$getBlock(blockHash); + const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); diff --git a/frontend/src/app/components/mining-dashboard/mining-dashboard.component.html b/frontend/src/app/components/mining-dashboard/mining-dashboard.component.html index 0e9ffb14d..63a93a3e4 100644 --- a/frontend/src/app/components/mining-dashboard/mining-dashboard.component.html +++ b/frontend/src/app/components/mining-dashboard/mining-dashboard.component.html @@ -5,30 +5,32 @@
Reward stats
-
-
-
-
-
Miners Reward
-
- -
in the last 8 blocks
+
+
+
+
+
+
Miners Reward
+
+ +
in the last 8 blocks
+
-
-
-
Reward Per Tx
-
- {{ rewardStats.rewardPerTx | amountShortener }} - sats/tx -
in the last 8 blocks
+
+
Reward Per Tx
+
+ {{ rewardStats.rewardPerTx | amountShortener }} + sats/tx +
in the last 8 blocks
+
-
-
-
Average Fee
-
- {{ rewardStats.feePerTx | amountShortener}} - sats/tx -
in the last 8 blocks
+
+
Average Fee
+
+ {{ rewardStats.feePerTx | amountShortener}} + sats/tx +
in the last 8 blocks
+
@@ -36,24 +38,24 @@
-
+
-
Miners Reward
-
+
Miners Reward
+
-
Reward Per Tx
-
+
Reward Per Tx
+
-
Average Fee
-
+
Average Fee
+
@@ -136,4 +138,4 @@
-
+
\ No newline at end of file diff --git a/frontend/src/app/components/mining-dashboard/mining-dashboard.component.scss b/frontend/src/app/components/mining-dashboard/mining-dashboard.component.scss index 148d589c6..a345be972 100644 --- a/frontend/src/app/components/mining-dashboard/mining-dashboard.component.scss +++ b/frontend/src/app/components/mining-dashboard/mining-dashboard.component.scss @@ -59,50 +59,57 @@ padding-bottom: 3px; } -.fee-estimation-container { +.reward-container { display: flex; - justify-content: space-between; - @media (min-width: 376px) { - flex-direction: row; + flex-direction: row; + justify-content: space-around; + height: 76px; + .shared-block { + color: #ffffff66; + font-size: 12px; } .item { - max-width: 150px; - margin: 0; - width: -webkit-fill-available; - @media (min-width: 376px) { - margin: 0 auto 0px; - } - &:first-child{ + display: table-cell; + padding: 0 5px; + width: 100%; + &:nth-child(1) { display: none; @media (min-width: 485px) { - display: block; + display: table-cell; } @media (min-width: 768px) { display: none; } @media (min-width: 992px) { - display: block; + display: table-cell; } } - &:last-child { - margin-bottom: 0; - } - .card-text span { - color: #ffffff66; - font-size: 12px; - top: 0px; - } - .fee-text{ - border-bottom: 1px solid #ffffff1c; - width: fit-content; - margin: auto; - line-height: 1.45; - padding: 0px 2px; - } - .fiat { - display: block; - font-size: 14px !important; - } + } + .card-text { + font-size: 22px; + margin-top: -9px; + position: relative; + } + .card-text.skeleton { + margin-top: 0px; + } +} + +.more-padding { + padding: 18px; +} + +.card-wrapper { + .card { + height: auto !important; + } + .card-body { + display: flex; + flex: inherit; + text-align: center; + flex-direction: column; + justify-content: space-around; + padding: 22px 20px; } } diff --git a/frontend/src/app/components/mining-dashboard/mining-dashboard.component.ts b/frontend/src/app/components/mining-dashboard/mining-dashboard.component.ts index 606bac5f1..c05c951b3 100644 --- a/frontend/src/app/components/mining-dashboard/mining-dashboard.component.ts +++ b/frontend/src/app/components/mining-dashboard/mining-dashboard.component.ts @@ -36,8 +36,8 @@ export class MiningDashboardComponent implements OnInit { return { 'totalReward': totalReward, - 'rewardPerTx': totalReward / totalTx, - 'feePerTx': totalFee / totalTx, + 'rewardPerTx': Math.round(totalReward / totalTx), + 'feePerTx': Math.round(totalFee / totalTx), } }) ); From edddf25917816a837b8281cd834385903a01c348 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 13 Mar 2022 16:04:49 +0100 Subject: [PATCH 5/5] Remove unnecessary migration version 15 --- backend/src/api/database-migration.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index f68f7f17d..c9c1da8e8 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,7 +6,7 @@ import logger from '../logger'; const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 15; + private static currentVersion = 14; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -175,12 +175,6 @@ class DatabaseMigration { await this.$executeQuery(connection, 'ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"'); } - if (databaseSchemaVersion < 15 && isBitcoin === true) { - logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`); - await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index - await this.$executeQuery(connection, 'ALTER TABLE `hashrates` MODIFY `avg_hashrate` DOUBLE UNSIGNED NOT NULL DEFAULT "0"'); - } - connection.release(); } catch (e) { connection.release();