diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 07ef9a669..631940dd2 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -19,6 +19,9 @@ import HashratesRepository from '../repositories/HashratesRepository'; import indexer from '../indexer'; import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; +import mining from './mining'; +import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; +import difficultyAdjustment from './difficulty-adjustment'; class Blocks { private blocks: BlockExtended[] = []; @@ -292,7 +295,8 @@ class Blocks { } logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`); } catch (e) { - logger.err(`Blocks summaries indexing failed. Reason: ${(e instanceof Error ? e.message : e)}`); + logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`); + throw e; } } @@ -364,18 +368,12 @@ class Blocks { logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`); loadingIndicators.setProgress('block-indexing', 100); } catch (e) { - logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e)); loadingIndicators.setProgress('block-indexing', 100); - return false; + throw e; } - const chainValid = await BlocksRepository.$validateChain(); - if (!chainValid) { - indexer.reindex(); - return false; - } - - return true; + return await BlocksRepository.$validateChain(); } public async $updateBlocks() { @@ -445,7 +443,10 @@ class Blocks { const newBlock = await this.$indexBlock(lastBlock['height'] - i); await this.$getStrippedBlockTransactions(newBlock.id, true, true); } - logger.info(`Re-indexed 10 blocks and summaries`); + await mining.$indexDifficultyAdjustments(); + await DifficultyAdjustmentsRepository.$deleteLastAdjustment(); + logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`); + indexer.reindex(); } await blocksRepository.$saveBlockInDatabase(blockExtended); @@ -457,6 +458,15 @@ class Blocks { } if (block.height % 2016 === 0) { + if (Common.indexingEnabled()) { + await DifficultyAdjustmentsRepository.$saveAdjustments({ + time: block.timestamp, + height: block.height, + difficulty: block.difficulty, + adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise + }); + } + this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100; this.lastDifficultyAdjustmentTime = block.timestamp; this.currentDifficulty = block.difficulty; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index f0f375309..8d9959cf2 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 21; + private static currentVersion = 22; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -226,6 +226,11 @@ class DatabaseMigration { await this.$executeQuery('DROP TABLE IF EXISTS `rates`'); await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices')); } + + if (databaseSchemaVersion < 22 && isBitcoin === true) { + await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`'); + await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments')); + } } catch (e) { throw e; } @@ -513,7 +518,7 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } - private getCreateRatesTableQuery(): string { + private getCreateRatesTableQuery(): string { // This table has been replaced by the prices table return `CREATE TABLE IF NOT EXISTS rates ( height int(10) unsigned NOT NULL, bisq_rates JSON NOT NULL, @@ -539,6 +544,17 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateDifficultyAdjustmentsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS difficulty_adjustments ( + time timestamp NOT NULL, + height int(10) unsigned NOT NULL, + difficulty double unsigned NOT NULL, + adjustment float NOT NULL, + PRIMARY KEY (height), + INDEX (time) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + public async $truncateIndexedData(tables: string[]) { const allowedTables = ['blocks', 'hashrates', 'prices']; diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 977cbe4e8..d69ff5cd9 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -1,4 +1,4 @@ -import { PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces'; +import { IndexedDifficultyAdjustment, PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces'; import BlocksRepository from '../repositories/BlocksRepository'; import PoolsRepository from '../repositories/PoolsRepository'; import HashratesRepository from '../repositories/HashratesRepository'; @@ -7,6 +7,8 @@ import logger from '../logger'; import { Common } from './common'; import loadingIndicators from './loading-indicators'; import { escape } from 'mysql2'; +import indexer from '../indexer'; +import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; class Mining { constructor() { @@ -262,6 +264,7 @@ class Mining { loadingIndicators.setProgress('weekly-hashrate-indexing', 100); } catch (e) { loadingIndicators.setProgress('weekly-hashrate-indexing', 100); + logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`); throw e; } } @@ -373,10 +376,53 @@ class Mining { loadingIndicators.setProgress('daily-hashrate-indexing', 100); } catch (e) { loadingIndicators.setProgress('daily-hashrate-indexing', 100); + logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`); throw e; } } + /** + * Index difficulty adjustments + */ + public async $indexDifficultyAdjustments(): Promise { + const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights(); + const indexedHeights = {}; + for (const height of indexedHeightsArray) { + indexedHeights[height] = true; + } + + const blocks: any = await BlocksRepository.$getBlocksDifficulty(); + + let currentDifficulty = 0; + let totalIndexed = 0; + + for (const block of blocks) { + if (block.difficulty !== currentDifficulty) { + if (block.height === 0 || indexedHeights[block.height] === true) { // Already indexed + currentDifficulty = block.difficulty; + continue; + } + + let adjustment = block.difficulty / Math.max(1, currentDifficulty); + adjustment = Math.round(adjustment * 1000000) / 1000000; // Remove float point noise + + await DifficultyAdjustmentsRepository.$saveAdjustments({ + time: block.time, + height: block.height, + difficulty: block.difficulty, + adjustment: adjustment, + }); + + totalIndexed++; + currentDifficulty = block.difficulty; + } + } + + if (totalIndexed > 0) { + logger.notice(`Indexed ${totalIndexed} difficulty adjustments`); + } + } + private getDateMidnight(date: Date): Date { date.setUTCHours(0); date.setUTCMinutes(0); diff --git a/backend/src/index.ts b/backend/src/index.ts index c4e80fd55..28215945f 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -290,6 +290,7 @@ class Server { .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', routes.$getDifficultyAdjustments) ; } diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 6ebba5bc4..8e4e7e87f 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -39,17 +39,21 @@ class Indexer { const chainValid = await blocks.$generateBlockDatabase(); if (chainValid === false) { // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration + logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`); + setTimeout(() => this.reindex(), 10000); this.indexerRunning = false; return; } + await mining.$indexDifficultyAdjustments(); await this.$resetHashratesIndexingState(); await mining.$generateNetworkHashrateHistory(); await mining.$generatePoolHashrateHistory(); await blocks.$generateBlocksSummariesDatabase(); } catch (e) { - this.reindex(); - logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e)); + this.indexerRunning = false; + logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e)); + setTimeout(() => this.reindex(), 10000); } this.indexerRunning = false; @@ -61,6 +65,7 @@ class Indexer { await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0); } catch (e) { logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; } } } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index a35dc6d76..983434564 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -224,6 +224,13 @@ export interface IDifficultyAdjustment { timeOffset: number; } +export interface IndexedDifficultyAdjustment { + time: number; // UNIX timestamp + height: number; // Block height + difficulty: number; + adjustment: number; +} + export interface RewardStats { totalReward: number; totalFee: number; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 01b7622f3..e88ac7877 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -7,6 +7,7 @@ import PoolsRepository from './PoolsRepository'; import HashratesRepository from './HashratesRepository'; import { escape } from 'mysql2'; import BlocksSummariesRepository from './BlocksSummariesRepository'; +import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository'; class BlocksRepository { /** @@ -381,48 +382,9 @@ class BlocksRepository { /** * Return blocks difficulty */ - public async $getBlocksDifficulty(interval: string | null): Promise { - interval = Common.getSqlInterval(interval); - - // :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162 - // Basically, using temporary user defined fields, we are able to extract all - // difficulty adjustments from the blocks tables. - // This allow use to avoid indexing it in another table. - let query = ` - SELECT - * - FROM - ( - SELECT - UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, height, - IF(@prevStatus = YT.difficulty, @rn := @rn + 1, - IF(@prevStatus := YT.difficulty, @rn := 1, @rn := 1) - ) AS rn - FROM blocks YT - CROSS JOIN - ( - SELECT @prevStatus := -1, @rn := 1 - ) AS var - `; - - if (interval) { - query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`; - } - - query += ` - ORDER BY YT.height - ) AS t - WHERE t.rn = 1 - ORDER BY t.height - `; - + public async $getBlocksDifficulty(): Promise { try { - const [rows]: any[] = await DB.query(query); - - for (const row of rows) { - delete row['rn']; - } - + const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`); return rows; } catch (e) { logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e)); @@ -452,26 +414,6 @@ class BlocksRepository { } } - /* - * Check if the last 10 blocks chain is valid - */ - public async $validateRecentBlocks(): Promise { - try { - const [lastBlocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash FROM blocks ORDER BY height DESC LIMIT 10`); - - for (let i = 0; i < lastBlocks.length - 1; ++i) { - if (lastBlocks[i].previous_block_hash !== lastBlocks[i + 1].hash) { - logger.warn(`Chain divergence detected at block ${lastBlocks[i].height}, re-indexing most recent data`); - return false; - } - } - - return true; - } catch (e) { - return true; // Don't do anything if there is a db error - } - } - /** * Check if the chain of block hash is valid and delete data from the stale branch if needed */ @@ -498,6 +440,7 @@ class BlocksRepository { await this.$deleteBlocksFrom(blocks[idx - 1].height); await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height); await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800); + await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height); return false; } ++idx; diff --git a/backend/src/repositories/DifficultyAdjustmentsRepository.ts b/backend/src/repositories/DifficultyAdjustmentsRepository.ts new file mode 100644 index 000000000..76324b5e6 --- /dev/null +++ b/backend/src/repositories/DifficultyAdjustmentsRepository.ts @@ -0,0 +1,88 @@ +import { Common } from '../api/common'; +import DB from '../database'; +import logger from '../logger'; +import { IndexedDifficultyAdjustment } from '../mempool.interfaces'; + +class DifficultyAdjustmentsRepository { + public async $saveAdjustments(adjustment: IndexedDifficultyAdjustment): Promise { + if (adjustment.height === 1) { + return; + } + + try { + const query = `INSERT INTO difficulty_adjustments(time, height, difficulty, adjustment) VALUE (FROM_UNIXTIME(?), ?, ?, ?)`; + const params: any[] = [ + adjustment.time, + adjustment.height, + adjustment.difficulty, + adjustment.adjustment, + ]; + await DB.query(query, params); + } catch (e: any) { + if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart + logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`); + } else { + logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + } + + public async $getAdjustments(interval: string | null, descOrder: boolean = false): Promise { + interval = Common.getSqlInterval(interval); + + let query = `SELECT UNIX_TIMESTAMP(time) as time, height, difficulty, adjustment + FROM difficulty_adjustments`; + + if (interval) { + query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`; + } + + if (descOrder === true) { + query += ` ORDER BY time DESC`; + } else { + query += ` ORDER BY time`; + } + + try { + const [rows] = await DB.query(query); + return rows as IndexedDifficultyAdjustment[]; + } catch (e) { + logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + public async $getAdjustmentsHeights(): Promise { + try { + const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`); + return rows.map(block => block.height); + } catch (e: any) { + logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + + public async $deleteAdjustementsFromHeight(height: number): Promise { + try { + logger.info(`Delete newer difficulty adjustments from height ${height} from the database`); + await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]); + } catch (e: any) { + logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + + public async $deleteLastAdjustment(): Promise { + try { + logger.info(`Delete last difficulty adjustment from the database`); + await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`); + } catch (e: any) { + logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } +} + +export default new DifficultyAdjustmentsRepository(); + diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 67f402f7f..3676aa49a 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -26,6 +26,7 @@ import mining from './api/mining'; import BlocksRepository from './repositories/BlocksRepository'; import HashratesRepository from './repositories/HashratesRepository'; import difficultyAdjustment from './api/difficulty-adjustment'; +import DifficultyAdjustmentsRepository from './repositories/DifficultyAdjustmentsRepository'; class Routes { constructor() {} @@ -653,7 +654,7 @@ class Routes { try { const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval); - const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval); + const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, false); const blockCount = await BlocksRepository.$blockCount(null, null); res.header('Pragma', 'public'); res.header('Cache-control', 'public'); @@ -730,6 +731,18 @@ class Routes { } } + public async $getDifficultyAdjustments(req: Request, res: Response) { + try { + const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, true); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment])); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + public async getBlock(req: Request, res: Response) { try { const block = await blocks.$getBlock(req.params.hash); diff --git a/frontend/src/app/components/address/address.component.html b/frontend/src/app/components/address/address.component.html index 6111075e4..ed3b1bd7e 100644 --- a/frontend/src/app/components/address/address.component.html +++ b/frontend/src/app/components/address/address.component.html @@ -130,7 +130,7 @@ Error loading address data.
({{ error.error }}) - + There many transactions on this address, more than your backend can handle. See more on setting up a stronger backend.

diff --git a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html index 6da8008c6..2dbe4d569 100644 --- a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html +++ b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html @@ -35,7 +35,7 @@ -