Merge pull request #1817 from mempool/nymkappa/feature/block-fee-usd-chart

This commit is contained in:
wiz 2022-07-12 23:35:35 +02:00 committed by GitHub
commit 9a29b4adf3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 650 additions and 333 deletions

View file

@ -17,11 +17,11 @@ import { prepareBlock } from '../utils/blocks-utils';
import BlocksRepository from '../repositories/BlocksRepository';
import HashratesRepository from '../repositories/HashratesRepository';
import indexer from '../indexer';
import fiatConversion from './fiat-conversion';
import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import mining from './mining/mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import difficultyAdjustment from './difficulty-adjustment';
class Blocks {
private blocks: BlockExtended[] = [];
@ -150,6 +150,7 @@ class Blocks {
blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
blockExtended.extras.usd = fiatConversion.getConversionRates().USD;
if (block.height === 0) {
blockExtended.extras.medianFee = 0; // 50th percentiles

View file

@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 30;
private static currentVersion = 31;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -12,8 +12,6 @@ class DatabaseMigration {
private blocksTruncatedMessage = `'blocks' table has been truncated. Re-indexing from scratch.`;
private hashratesTruncatedMessage = `'hashrates' table has been truncated. Re-indexing from scratch.`;
constructor() { }
/**
* Avoid printing multiple time the same message
*/
@ -104,199 +102,193 @@ class DatabaseMigration {
await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
try {
await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
}
if (databaseSchemaVersion < 3) {
await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
}
if (databaseSchemaVersion < 4) {
await this.$executeQuery('DROP table IF EXISTS blocks;');
await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
}
if (databaseSchemaVersion < 5 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 6 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
// Cleanup original blocks fields type
await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
// We also fix the pools.id type so we need to drop/re-create the foreign key
await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
// Add new block indexing fields
await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
}
await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
}
if (databaseSchemaVersion < 3) {
await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
}
if (databaseSchemaVersion < 4) {
await this.$executeQuery('DROP table IF EXISTS blocks;');
await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
}
if (databaseSchemaVersion < 5 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 7 && isBitcoin === true) {
await this.$executeQuery('DROP table IF EXISTS hashrates;');
await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
}
if (databaseSchemaVersion < 6 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
// Cleanup original blocks fields type
await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
// We also fix the pools.id type so we need to drop/re-create the foreign key
await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
// Add new block indexing fields
await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
}
if (databaseSchemaVersion < 8 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
}
if (databaseSchemaVersion < 7 && isBitcoin === true) {
await this.$executeQuery('DROP table IF EXISTS hashrates;');
await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
}
if (databaseSchemaVersion < 9 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
}
if (databaseSchemaVersion < 8 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
}
if (databaseSchemaVersion < 10 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
}
if (databaseSchemaVersion < 9 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
}
if (databaseSchemaVersion < 11 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery(`ALTER TABLE blocks
ADD avg_fee INT UNSIGNED NULL,
ADD avg_fee_rate INT UNSIGNED NULL
`);
await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 10 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
}
if (databaseSchemaVersion < 12 && isBitcoin === true) {
// No need to re-index because the new data type can contain larger values
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 11 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery(`ALTER TABLE blocks
ADD avg_fee INT UNSIGNED NULL,
ADD avg_fee_rate INT UNSIGNED NULL
`);
await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 13 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 12 && isBitcoin === true) {
// No need to re-index because the new data type can contain larger values
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 14 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 13 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 16 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
}
if (databaseSchemaVersion < 14 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 17 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
}
if (databaseSchemaVersion < 16 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
}
if (databaseSchemaVersion < 18 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
}
if (databaseSchemaVersion < 17 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
}
if (databaseSchemaVersion < 19) {
await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
}
if (databaseSchemaVersion < 18 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
}
if (databaseSchemaVersion < 20 && isBitcoin === true) {
await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
}
if (databaseSchemaVersion < 19) {
await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
}
if (databaseSchemaVersion < 21) {
await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
}
if (databaseSchemaVersion < 20 && isBitcoin === true) {
await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
}
if (databaseSchemaVersion < 22 && isBitcoin === true) {
await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
}
if (databaseSchemaVersion < 21) {
await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
}
if (databaseSchemaVersion < 23) {
await this.$executeQuery('TRUNCATE `prices`');
await this.$executeQuery('ALTER TABLE `prices` DROP `avg_prices`');
await this.$executeQuery('ALTER TABLE `prices` ADD `USD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `EUR` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `GBP` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `CAD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
}
if (databaseSchemaVersion < 22 && isBitcoin === true) {
await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
}
if (databaseSchemaVersion < 24 && isBitcoin == true) {
await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
}
if (databaseSchemaVersion < 23) {
await this.$executeQuery('TRUNCATE `prices`');
await this.$executeQuery('ALTER TABLE `prices` DROP `avg_prices`');
await this.$executeQuery('ALTER TABLE `prices` ADD `USD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `EUR` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `GBP` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `CAD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
}
if (databaseSchemaVersion < 25 && isBitcoin === true) {
await this.$executeQuery(`INSERT INTO state VALUES('last_node_stats', 0, '1970-01-01');`);
await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats'));
await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats'));
}
if (databaseSchemaVersion < 24 && isBitcoin == true) {
await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
}
if (databaseSchemaVersion < 26 && isBitcoin === true) {
this.uniqueLog(logger.notice, `'lightning_stats' table has been truncated. Will re-generate historical data from scratch.`);
await this.$executeQuery(`TRUNCATE lightning_stats`);
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 25 && isBitcoin === true) {
await this.$executeQuery(`INSERT INTO state VALUES('last_node_stats', 0, '1970-01-01');`);
await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats'));
await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats'));
}
if (databaseSchemaVersion < 27 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_capacity bigint(20) unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_fee_rate int(11) unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_capacity bigint(20) unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_fee_rate int(11) unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 26 && isBitcoin === true) {
this.uniqueLog(logger.notice, `'lightning_stats' table has been truncated. Will re-generate historical data from scratch.`);
await this.$executeQuery(`TRUNCATE lightning_stats`);
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 28 && isBitcoin === true) {
await this.$executeQuery(`TRUNCATE lightning_stats`);
await this.$executeQuery(`TRUNCATE node_stats`);
await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
}
if (databaseSchemaVersion < 28 && isBitcoin === true) {
await this.$executeQuery(`TRUNCATE lightning_stats`);
await this.$executeQuery(`TRUNCATE node_stats`);
await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
}
if (databaseSchemaVersion < 29 && isBitcoin === true) {
await this.$executeQuery(this.getCreateGeoNamesTableQuery(), await this.$checkIfTableExists('geo_names'));
await this.$executeQuery('ALTER TABLE `nodes` ADD as_number int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD city_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD country_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD accuracy_radius int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL');
}
if (databaseSchemaVersion < 29 && isBitcoin === true) {
await this.$executeQuery(this.getCreateGeoNamesTableQuery(), await this.$checkIfTableExists('geo_names'));
await this.$executeQuery('ALTER TABLE `nodes` ADD as_number int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD city_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD country_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD accuracy_radius int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL');
}
if (databaseSchemaVersion < 30 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization") NOT NULL');
}
if (databaseSchemaVersion < 30 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization") NOT NULL');
}
if (databaseSchemaVersion < 31 && isBitcoin == true) { // Link blocks to prices
await this.$executeQuery('ALTER TABLE `prices` ADD `id` int NULL AUTO_INCREMENT UNIQUE');
await this.$executeQuery('DROP TABLE IF EXISTS `blocks_prices`');
await this.$executeQuery(this.getCreateBlocksPricesTableQuery(), await this.$checkIfTableExists('blocks_prices'));
} catch (e) {
throw e;
}
}
@ -335,7 +327,7 @@ class DatabaseMigration {
/**
* Small query execution wrapper to log all executed queries
*/
private async $executeQuery(query: string, silent: boolean = false): Promise<any> {
private async $executeQuery(query: string, silent = false): Promise<any> {
if (!silent) {
logger.debug('MIGRATIONS: Execute query:\n' + query);
}
@ -364,21 +356,17 @@ class DatabaseMigration {
* Create the `state` table
*/
private async $createMigrationStateTable(): Promise<void> {
try {
const query = `CREATE TABLE IF NOT EXISTS state (
name varchar(25) NOT NULL,
number int(11) NULL,
string varchar(100) NULL,
CONSTRAINT name_unique UNIQUE (name)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
await this.$executeQuery(query);
const query = `CREATE TABLE IF NOT EXISTS state (
name varchar(25) NOT NULL,
number int(11) NULL,
string varchar(100) NULL,
CONSTRAINT name_unique UNIQUE (name)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
await this.$executeQuery(query);
// Set initial values
await this.$executeQuery(`INSERT INTO state VALUES('schema_version', 0, NULL);`);
await this.$executeQuery(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`);
} catch (e) {
throw e;
}
// Set initial values
await this.$executeQuery(`INSERT INTO state VALUES('schema_version', 0, NULL);`);
await this.$executeQuery(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`);
}
/**
@ -718,6 +706,15 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
}
private getCreateBlocksPricesTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS blocks_prices (
height int(10) unsigned NOT NULL,
price_id int(10) unsigned NOT NULL,
PRIMARY KEY (height),
INDEX (price_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates', 'prices'];

View file

@ -1,4 +1,4 @@
import { IndexedDifficultyAdjustment, PoolInfo, PoolStats, RewardStats } from '../../mempool.interfaces';
import { BlockPrice, PoolInfo, PoolStats, RewardStats } from '../../mempool.interfaces';
import BlocksRepository from '../../repositories/BlocksRepository';
import PoolsRepository from '../../repositories/PoolsRepository';
import HashratesRepository from '../../repositories/HashratesRepository';
@ -7,12 +7,14 @@ import logger from '../../logger';
import { Common } from '../common';
import loadingIndicators from '../loading-indicators';
import { escape } from 'mysql2';
import indexer from '../../indexer';
import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
import config from '../../config';
import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
import PricesRepository from '../../repositories/PricesRepository';
class Mining {
blocksPriceIndexingRunning = false;
constructor() {
}
@ -31,7 +33,7 @@ class Mining {
*/
public async $getHistoricalBlockFees(interval: string | null = null): Promise<any> {
return await BlocksRepository.$getHistoricalBlockFees(
this.getTimeRange(interval),
this.getTimeRange(interval, 5),
Common.getSqlInterval(interval)
);
}
@ -453,6 +455,70 @@ class Mining {
}
}
/**
* Create a link between blocks and the latest price at when they were mined
*/
public async $indexBlockPrices() {
if (this.blocksPriceIndexingRunning === true) {
return;
}
this.blocksPriceIndexingRunning = true;
try {
const prices: any[] = await PricesRepository.$getPricesTimesAndId();
const blocksWithoutPrices: any[] = await BlocksRepository.$getBlocksWithoutPrice();
let totalInserted = 0;
const blocksPrices: BlockPrice[] = [];
for (const block of blocksWithoutPrices) {
// Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks
if (block.height < 68951) {
blocksPrices.push({
height: block.height,
priceId: prices[0].id,
});
continue;
}
for (const price of prices) {
if (block.timestamp < price.time) {
blocksPrices.push({
height: block.height,
priceId: price.id,
});
break;
};
}
if (blocksPrices.length >= 100000) {
totalInserted += blocksPrices.length;
if (blocksWithoutPrices.length > 200000) {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
} else {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
}
await BlocksRepository.$saveBlockPrices(blocksPrices);
blocksPrices.length = 0;
}
}
if (blocksPrices.length > 0) {
totalInserted += blocksPrices.length;
if (blocksWithoutPrices.length > 200000) {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
} else {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
}
await BlocksRepository.$saveBlockPrices(blocksPrices);
}
} catch (e) {
this.blocksPriceIndexingRunning = false;
throw e;
}
this.blocksPriceIndexingRunning = false;
}
private getDateMidnight(date: Date): Date {
date.setUTCHours(0);
date.setUTCMinutes(0);
@ -462,18 +528,18 @@ class Mining {
return date;
}
private getTimeRange(interval: string | null): number {
private getTimeRange(interval: string | null, scale = 1): number {
switch (interval) {
case '3y': return 43200; // 12h
case '2y': return 28800; // 8h
case '1y': return 28800; // 8h
case '6m': return 10800; // 3h
case '3m': return 7200; // 2h
case '1m': return 1800; // 30min
case '1w': return 300; // 5min
case '3d': return 1;
case '24h': return 1;
default: return 86400; // 24h
case '3y': return 43200 * scale; // 12h
case '2y': return 28800 * scale; // 8h
case '1y': return 28800 * scale; // 8h
case '6m': return 10800 * scale; // 3h
case '3m': return 7200 * scale; // 2h
case '1m': return 1800 * scale; // 30min
case '1w': return 300 * scale; // 5min
case '3d': return 1 * scale;
case '24h': return 1 * scale;
default: return 86400 * scale;
}
}
}

View file

@ -24,7 +24,6 @@ import icons from './api/liquid/icons';
import { Common } from './api/common';
import poolsUpdater from './tasks/pools-updater';
import indexer from './indexer';
import priceUpdater from './tasks/price-updater';
import nodesRoutes from './api/explorer/nodes.routes';
import channelsRoutes from './api/explorer/channels.routes';
import generalLightningRoutes from './api/explorer/general.routes';
@ -166,7 +165,6 @@ class Server {
await blocks.$updateBlocks();
await memPool.$updateMempool();
indexer.$run();
priceUpdater.$run();
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
this.currentBackendRetryInterval = 5;

View file

@ -5,6 +5,7 @@ import mining from './api/mining/mining';
import logger from './logger';
import HashratesRepository from './repositories/HashratesRepository';
import bitcoinClient from './api/bitcoin/bitcoin-client';
import priceUpdater from './tasks/price-updater';
class Indexer {
runIndexer = true;
@ -38,6 +39,8 @@ class Indexer {
logger.debug(`Running mining indexer`);
try {
await priceUpdater.$run();
const chainValid = await blocks.$generateBlockDatabase();
if (chainValid === false) {
// Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
@ -47,8 +50,9 @@ class Indexer {
return;
}
await mining.$indexBlockPrices();
await mining.$indexDifficultyAdjustments();
await this.$resetHashratesIndexingState();
await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();
await blocks.$generateBlocksSummariesDatabase();

View file

@ -109,6 +109,7 @@ export interface BlockExtension {
avgFee?: number;
avgFeeRate?: number;
coinbaseRaw?: string;
usd?: number | null;
}
export interface BlockExtended extends IEsploraApi.Block {
@ -120,6 +121,11 @@ export interface BlockSummary {
transactions: TransactionStripped[];
}
export interface BlockPrice {
height: number;
priceId: number;
}
export interface TransactionMinerInfo {
vin: VinStrippedToScriptsig[];
vout: VoutStrippedToScriptPubkey[];

View file

@ -1,4 +1,4 @@
import { BlockExtended } from '../mempool.interfaces';
import { BlockExtended, BlockPrice } from '../mempool.interfaces';
import DB from '../database';
import logger from '../logger';
import { Common } from '../api/common';
@ -256,7 +256,7 @@ class BlocksRepository {
const params: any[] = [];
let query = ` SELECT
height,
blocks.height,
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
size,
@ -308,7 +308,7 @@ class BlocksRepository {
public async $getBlockByHeight(height: number): Promise<object | null> {
try {
const [rows]: any[] = await DB.query(`SELECT
height,
blocks.height,
hash,
hash as id,
UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
@ -336,7 +336,7 @@ class BlocksRepository {
avg_fee_rate
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE height = ${height};
WHERE blocks.height = ${height}
`);
if (rows.length <= 0) {
@ -357,15 +357,15 @@ class BlocksRepository {
public async $getBlockByHash(hash: string): Promise<object | null> {
try {
const query = `
SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
pools.addresses as pool_addresses, pools.regexes as pool_regexes,
previous_block_hash as previousblockhash
FROM blocks
JOIN pools ON blocks.pool_id = pools.id
WHERE hash = '${hash}';
WHERE hash = ?;
`;
const [rows]: any[] = await DB.query(query);
const [rows]: any[] = await DB.query(query, [hash]);
if (rows.length <= 0) {
return null;
@ -387,7 +387,20 @@ class BlocksRepository {
const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`);
return rows;
} catch (e) {
logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot get blocks difficulty list from the db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Return blocks height
*/
public async $getBlocksHeightsAndTimestamp(): Promise<object[]> {
try {
const [rows]: any[] = await DB.query(`SELECT height, blockTimestamp as timestamp FROM blocks`);
return rows;
} catch (e) {
logger.err('Cannot get blocks height and timestamp from the db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
@ -473,10 +486,14 @@ class BlocksRepository {
public async $getHistoricalBlockFees(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT
CAST(AVG(height) as INT) as avgHeight,
CAST(AVG(blocks.height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(fees) as INT) as avgFees
FROM blocks`;
CAST(AVG(fees) as INT) as avgFees,
prices.USD
FROM blocks
JOIN blocks_prices on blocks_prices.height = blocks.height
JOIN prices on prices.id = blocks_prices.price_id
`;
if (interval !== null) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
@ -498,10 +515,14 @@ class BlocksRepository {
public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT
CAST(AVG(height) as INT) as avgHeight,
CAST(AVG(blocks.height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(reward) as INT) as avgRewards
FROM blocks`;
CAST(AVG(reward) as INT) as avgRewards,
prices.USD
FROM blocks
JOIN blocks_prices on blocks_prices.height = blocks.height
JOIN prices on prices.id = blocks_prices.price_id
`;
if (interval !== null) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
@ -628,6 +649,46 @@ class BlocksRepository {
throw e;
}
}
/**
* Get all blocks which have not be linked to a price yet
*/
public async $getBlocksWithoutPrice(): Promise<object[]> {
try {
const [rows]: any[] = await DB.query(`
SELECT UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.height
FROM blocks
LEFT JOIN blocks_prices ON blocks.height = blocks_prices.height
WHERE blocks_prices.height IS NULL
ORDER BY blocks.height
`);
return rows;
} catch (e) {
logger.err('Cannot get blocks height and timestamp from the db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Save block price by batch
*/
public async $saveBlockPrices(blockPrices: BlockPrice[]): Promise<void> {
try {
let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
for (const price of blockPrices) {
query += ` (${price.height}, ${price.priceId}),`
}
query = query.slice(0, -1);
await DB.query(query);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save blocks prices for blocks [${blockPrices[0].height} to ${blockPrices[blockPrices.length - 1].height}] because it has already been indexed, ignoring`);
} else {
logger.err(`Cannot save blocks prices for blocks [${blockPrices[0].height} to ${blockPrices[blockPrices.length - 1].height}] into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
}
export default new BlocksRepository();

View file

@ -33,9 +33,14 @@ class PricesRepository {
}
public async $getPricesTimes(): Promise<number[]> {
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1`);
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time`);
return times.map(time => time.time);
}
public async $getPricesTimesAndId(): Promise<number[]> {
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time, id, USD from prices ORDER BY time`);
return times;
}
}
export default new PricesRepository();

View file

@ -16,7 +16,7 @@ class BitfinexApi implements PriceFeed {
return response ? parseInt(response['last_price'], 10) : -1;
}
public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
const priceHistory: PriceHistory = {};
for (const currency of currencies) {
@ -24,7 +24,7 @@ class BitfinexApi implements PriceFeed {
continue;
}
const response = await query(this.urlHist.replace('{GRANULARITY}', '1h').replace('{CURRENCY}', currency));
const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '1h' : '1D').replace('{CURRENCY}', currency));
const pricesRaw = response ? response : [];
for (const price of pricesRaw as any[]) {

View file

@ -16,7 +16,7 @@ class BitflyerApi implements PriceFeed {
return response ? parseInt(response['ltp'], 10) : -1;
}
public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
return [];
}
}

View file

@ -16,7 +16,7 @@ class CoinbaseApi implements PriceFeed {
return response ? parseInt(response['data']['amount'], 10) : -1;
}
public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
const priceHistory: PriceHistory = {};
for (const currency of currencies) {
@ -24,7 +24,7 @@ class CoinbaseApi implements PriceFeed {
continue;
}
const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency));
const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '3600' : '86400').replace('{CURRENCY}', currency));
const pricesRaw = response ? response : [];
for (const price of pricesRaw as any[]) {

View file

@ -16,7 +16,7 @@ class FtxApi implements PriceFeed {
return response ? parseInt(response['result']['last'], 10) : -1;
}
public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
const priceHistory: PriceHistory = {};
for (const currency of currencies) {
@ -24,7 +24,7 @@ class FtxApi implements PriceFeed {
continue;
}
const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency));
const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '3600' : '86400').replace('{CURRENCY}', currency));
const pricesRaw = response ? response['result'] : [];
for (const price of pricesRaw as any[]) {

View file

@ -16,7 +16,7 @@ class GeminiApi implements PriceFeed {
return response ? parseInt(response['last'], 10) : -1;
}
public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
const priceHistory: PriceHistory = {};
for (const currency of currencies) {
@ -24,7 +24,7 @@ class GeminiApi implements PriceFeed {
continue;
}
const response = await query(this.urlHist.replace('{GRANULARITY}', '1hr').replace('{CURRENCY}', currency));
const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '1hr' : '1day').replace('{CURRENCY}', currency));
const pricesRaw = response ? response : [];
for (const price of pricesRaw as any[]) {

View file

@ -26,7 +26,7 @@ class KrakenApi implements PriceFeed {
return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1;
}
public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
const priceHistory: PriceHistory = {};
for (const currency of currencies) {

View file

@ -16,7 +16,7 @@ export interface PriceFeed {
currencies: string[];
$fetchPrice(currency): Promise<number>;
$fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory>;
$fetchRecentPrice(currencies: string[], type: string): Promise<PriceHistory>;
}
export interface PriceHistory {
@ -185,7 +185,8 @@ class PriceUpdater {
await new KrakenApi().$insertHistoricalPrice();
// Insert missing recent hourly prices
await this.$insertMissingRecentPrices();
await this.$insertMissingRecentPrices('day');
await this.$insertMissingRecentPrices('hour');
this.historyInserted = true;
this.lastHistoricalRun = new Date().getTime();
@ -195,17 +196,17 @@ class PriceUpdater {
* Find missing hourly prices and insert them in the database
* It has a limited backward range and it depends on which API are available
*/
private async $insertMissingRecentPrices(): Promise<void> {
private async $insertMissingRecentPrices(type: 'hour' | 'day'): Promise<void> {
const existingPriceTimes = await PricesRepository.$getPricesTimes();
logger.info(`Fetching hourly price history from exchanges and saving missing ones into the database, this may take a while`);
logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database, this may take a while`);
const historicalPrices: PriceHistory[] = [];
// Fetch all historical hourly prices
for (const feed of this.feeds) {
try {
historicalPrices.push(await feed.$fetchRecentHourlyPrice(this.currencies));
historicalPrices.push(await feed.$fetchRecentPrice(this.currencies, type));
} catch (e) {
logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
}
@ -252,9 +253,9 @@ class PriceUpdater {
}
if (totalInserted > 0) {
logger.notice(`Inserted ${totalInserted} hourly historical prices into the db`);
logger.notice(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`);
} else {
logger.debug(`Inserted ${totalInserted} hourly historical prices into the db`);
logger.debug(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`);
}
}
}

View file

@ -27,6 +27,7 @@ export function prepareBlock(block: any): BlockExtended {
name: block.pool_name,
slug: block.pool_slug,
} : undefined),
usd: block?.extras?.usd ?? block.usd ?? null,
}
};
}

View file

@ -13,6 +13,7 @@ import { SharedModule } from './shared/shared.module';
import { StorageService } from './services/storage.service';
import { HttpCacheInterceptor } from './services/http-cache.interceptor';
import { LanguageService } from './services/language.service';
import { FiatShortenerPipe } from './shared/pipes/fiat-shortener.pipe';
import { ShortenStringPipe } from './shared/pipes/shorten-string-pipe/shorten-string.pipe';
import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe';
@ -37,6 +38,7 @@ import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe
StorageService,
LanguageService,
ShortenStringPipe,
FiatShortenerPipe,
CapAddressPipe,
{ provide: HTTP_INTERCEPTORS, useClass: HttpCacheInterceptor, multi: true }
],

View file

@ -180,8 +180,8 @@ export class BlockFeeRatesGraphComponent implements OnInit {
}
let tooltip = `<b style="color: white; margin-left: 2px">${formatterXAxis(this.locale, this.timespan, parseInt(data[0].axisValue, 10))}</b><br>`;
for (const pool of data.reverse()) {
tooltip += `${pool.marker} ${pool.seriesName}: ${pool.data[1]} sats/vByte<br>`;
for (const rate of data.reverse()) {
tooltip += `${rate.marker} ${rate.seriesName}: ${rate.data[1]} sats/vByte<br>`;
}
if (['24h', '3d'].includes(this.timespan)) {

View file

@ -8,15 +8,6 @@
</button>
<form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
<div class="btn-group btn-group-toggle" ngbRadioGroup name="radioBasic" formControlName="dateSpan">
<label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 144">
<input ngbButton type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 24h
</label>
<label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 432">
<input ngbButton type="radio" [value]="'3d'" fragment="3d" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 3D
</label>
<label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 1008">
<input ngbButton type="radio" [value]="'1w'" fragment="1w" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 1W
</label>
<label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 4320">
<input ngbButton type="radio" [value]="'1m'" fragment="1m" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 1M
</label>

View file

@ -4,12 +4,13 @@ import { Observable } from 'rxjs';
import { map, share, startWith, switchMap, tap } from 'rxjs/operators';
import { ApiService } from 'src/app/services/api.service';
import { SeoService } from 'src/app/services/seo.service';
import { formatNumber } from '@angular/common';
import { formatCurrency, formatNumber, getCurrencySymbol } from '@angular/common';
import { FormBuilder, FormGroup } from '@angular/forms';
import { download, formatterXAxis, formatterXAxisLabel } from 'src/app/shared/graphs.utils';
import { download, formatterXAxis, formatterXAxisLabel, formatterXAxisTimeCategory } from 'src/app/shared/graphs.utils';
import { StorageService } from 'src/app/services/storage.service';
import { MiningService } from 'src/app/services/mining.service';
import { ActivatedRoute } from '@angular/router';
import { FiatShortenerPipe } from 'src/app/shared/pipes/fiat-shortener.pipe';
@Component({
selector: 'app-block-fees-graph',
@ -51,6 +52,7 @@ export class BlockFeesGraphComponent implements OnInit {
private storageService: StorageService,
private miningService: MiningService,
private route: ActivatedRoute,
private fiatShortenerPipe: FiatShortenerPipe,
) {
this.radioGroupForm = this.formBuilder.group({ dateSpan: '1y' });
this.radioGroupForm.controls.dateSpan.setValue('1y');
@ -58,14 +60,14 @@ export class BlockFeesGraphComponent implements OnInit {
ngOnInit(): void {
this.seoService.setTitle($localize`:@@6c453b11fd7bd159ae30bc381f367bc736d86909:Block Fees`);
this.miningWindowPreference = this.miningService.getDefaultTimespan('24h');
this.miningWindowPreference = this.miningService.getDefaultTimespan('1m');
this.radioGroupForm = this.formBuilder.group({ dateSpan: this.miningWindowPreference });
this.radioGroupForm.controls.dateSpan.setValue(this.miningWindowPreference);
this.route
.fragment
.subscribe((fragment) => {
if (['24h', '3d', '1w', '1m', '3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) {
if (['1m', '3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) {
this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false });
}
});
@ -82,6 +84,7 @@ export class BlockFeesGraphComponent implements OnInit {
tap((response) => {
this.prepareChartOptions({
blockFees: response.body.map(val => [val.timestamp * 1000, val.avgFees / 100000000, val.avgHeight]),
blockFeesUSD: response.body.filter(val => val.USD > 0).map(val => [val.timestamp * 1000, val.avgFees / 100000000 * val.USD, val.avgHeight]),
});
this.isLoading = false;
}),
@ -97,17 +100,32 @@ export class BlockFeesGraphComponent implements OnInit {
}
prepareChartOptions(data) {
let title: object;
if (data.blockFees.length === 0) {
title = {
textStyle: {
color: 'grey',
fontSize: 15
},
text: $localize`:@@23555386d8af1ff73f297e89dd4af3f4689fb9dd:Indexing blocks`,
left: 'center',
top: 'center'
};
}
this.chartOptions = {
animation: false,
title: title,
color: [
new graphic.LinearGradient(0, 0, 0, 0.65, [
{ offset: 0, color: '#F4511E' },
{ offset: 0.25, color: '#FB8C00' },
{ offset: 0.5, color: '#FFB300' },
{ offset: 0.75, color: '#FDD835' },
{ offset: 1, color: '#7CB342' }
new graphic.LinearGradient(0, 0, 0, 1, [
{ offset: 0, color: '#FDD835' },
{ offset: 1, color: '#FB8C00' },
]),
new graphic.LinearGradient(0, 0, 0, 1, [
{ offset: 0, color: '#C0CA33' },
{ offset: 1, color: '#1B5E20' },
]),
],
animation: false,
grid: {
top: 30,
bottom: 80,
@ -128,30 +146,54 @@ export class BlockFeesGraphComponent implements OnInit {
align: 'left',
},
borderColor: '#000',
formatter: (ticks) => {
let tooltip = `<b style="color: white; margin-left: 2px">${formatterXAxis(this.locale, this.timespan, parseInt(ticks[0].axisValue, 10))}</b><br>`;
tooltip += `${ticks[0].marker} ${ticks[0].seriesName}: ${formatNumber(ticks[0].data[1], this.locale, '1.3-3')} BTC`;
tooltip += `<br>`;
formatter: function (data) {
if (data.length <= 0) {
return '';
}
let tooltip = `<b style="color: white; margin-left: 2px">
${formatterXAxis(this.locale, this.timespan, parseInt(data[0].axisValue, 10))}</b><br>`;
if (['24h', '3d'].includes(this.timespan)) {
tooltip += `<small>` + $localize`At block: ${ticks[0].data[2]}` + `</small>`;
} else {
tooltip += `<small>` + $localize`Around block: ${ticks[0].data[2]}` + `</small>`;
for (const tick of data) {
if (tick.seriesIndex === 0) {
tooltip += `${tick.marker} ${tick.seriesName}: ${formatNumber(tick.data[1], this.locale, '1.3-3')} BTC<br>`;
} else if (tick.seriesIndex === 1) {
tooltip += `${tick.marker} ${tick.seriesName}: ${formatCurrency(tick.data[1], this.locale, getCurrencySymbol('USD', 'narrow'), 'USD', '1.0-0')}<br>`;
}
}
tooltip += `<small>* On average around block ${data[0].data[2]}</small>`;
return tooltip;
}
}.bind(this)
},
xAxis: {
name: formatterXAxisLabel(this.locale, this.timespan),
nameLocation: 'middle',
nameTextStyle: {
padding: [10, 0, 0, 0],
},
xAxis: data.blockFees.length === 0 ? undefined :
{
type: 'time',
splitNumber: this.isMobile() ? 5 : 10,
axisLabel: {
hideOverlap: true,
}
},
yAxis: [
legend: data.blockFees.length === 0 ? undefined : {
data: [
{
name: 'Fees BTC',
inactiveColor: 'rgb(110, 112, 121)',
textStyle: {
color: 'white',
},
icon: 'roundRect',
},
{
name: 'Fees USD',
inactiveColor: 'rgb(110, 112, 121)',
textStyle: {
color: 'white',
},
icon: 'roundRect',
},
],
},
yAxis: data.blockFees.length === 0 ? undefined : [
{
type: 'value',
axisLabel: {
@ -168,21 +210,51 @@ export class BlockFeesGraphComponent implements OnInit {
}
},
},
],
series: [
{
zlevel: 0,
name: $localize`:@@c20172223f84462032664d717d739297e5a9e2fe:Fees`,
showSymbol: false,
symbol: 'none',
data: data.blockFees,
type: 'line',
lineStyle: {
width: 2,
type: 'value',
position: 'right',
axisLabel: {
color: 'rgb(110, 112, 121)',
formatter: function(val) {
return this.fiatShortenerPipe.transform(val);
}.bind(this)
},
splitLine: {
show: false,
},
},
],
dataZoom: [{
series: data.blockFees.length === 0 ? undefined : [
{
legendHoverLink: false,
zlevel: 0,
yAxisIndex: 0,
name: 'Fees BTC',
data: data.blockFees,
type: 'line',
smooth: 0.25,
symbol: 'none',
lineStyle: {
width: 1,
opacity: 1,
}
},
{
legendHoverLink: false,
zlevel: 1,
yAxisIndex: 1,
name: 'Fees USD',
data: data.blockFeesUSD,
type: 'line',
smooth: 0.25,
symbol: 'none',
lineStyle: {
width: 2,
opacity: 1,
}
},
],
dataZoom: data.blockFees.length === 0 ? undefined : [{
type: 'inside',
realtime: true,
zoomLock: true,

View file

@ -9,15 +9,6 @@
</button>
<form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
<div class="btn-group btn-group-toggle" ngbRadioGroup name="radioBasic" formControlName="dateSpan">
<label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 144">
<input ngbButton type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 24h
</label>
<label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 432">
<input ngbButton type="radio" [value]="'3d'" fragment="3d" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 3D
</label>
<label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 1008">
<input ngbButton type="radio" [value]="'1w'" fragment="1w" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 1W
</label>
<label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 4320">
<input ngbButton type="radio" [value]="'1m'" fragment="1m" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 1M
</label>

View file

@ -4,12 +4,13 @@ import { Observable } from 'rxjs';
import { map, share, startWith, switchMap, tap } from 'rxjs/operators';
import { ApiService } from 'src/app/services/api.service';
import { SeoService } from 'src/app/services/seo.service';
import { formatNumber } from '@angular/common';
import { formatCurrency, formatNumber, getCurrencySymbol } from '@angular/common';
import { FormBuilder, FormGroup } from '@angular/forms';
import { download, formatterXAxis, formatterXAxisLabel } from 'src/app/shared/graphs.utils';
import { download, formatterXAxis, formatterXAxisLabel, formatterXAxisTimeCategory } from 'src/app/shared/graphs.utils';
import { MiningService } from 'src/app/services/mining.service';
import { StorageService } from 'src/app/services/storage.service';
import { ActivatedRoute } from '@angular/router';
import { FiatShortenerPipe } from 'src/app/shared/pipes/fiat-shortener.pipe';
@Component({
selector: 'app-block-rewards-graph',
@ -51,19 +52,20 @@ export class BlockRewardsGraphComponent implements OnInit {
private miningService: MiningService,
private storageService: StorageService,
private route: ActivatedRoute,
private fiatShortenerPipe: FiatShortenerPipe,
) {
}
ngOnInit(): void {
this.seoService.setTitle($localize`:@@8ba8fe810458280a83df7fdf4c614dfc1a826445:Block Rewards`);
this.miningWindowPreference = this.miningService.getDefaultTimespan('24h');
this.miningWindowPreference = this.miningService.getDefaultTimespan('3m');
this.radioGroupForm = this.formBuilder.group({ dateSpan: this.miningWindowPreference });
this.radioGroupForm.controls.dateSpan.setValue(this.miningWindowPreference);
this.route
.fragment
.subscribe((fragment) => {
if (['24h', '3d', '1w', '1m', '3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) {
if (['3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) {
this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false });
}
});
@ -80,6 +82,7 @@ export class BlockRewardsGraphComponent implements OnInit {
tap((response) => {
this.prepareChartOptions({
blockRewards: response.body.map(val => [val.timestamp * 1000, val.avgRewards / 100000000, val.avgHeight]),
blockRewardsUSD: response.body.filter(val => val.USD > 0).map(val => [val.timestamp * 1000, val.avgRewards / 100000000 * val.USD, val.avgHeight]),
});
this.isLoading = false;
}),
@ -95,15 +98,32 @@ export class BlockRewardsGraphComponent implements OnInit {
}
prepareChartOptions(data) {
let title: object;
if (data.blockRewards.length === 0) {
title = {
textStyle: {
color: 'grey',
fontSize: 15
},
text: $localize`:@@23555386d8af1ff73f297e89dd4af3f4689fb9dd:Indexing blocks`,
left: 'center',
top: 'center'
};
}
const scaleFactor = 0.1;
this.chartOptions = {
title: title,
animation: false,
color: [
new graphic.LinearGradient(0, 0, 0, 0.65, [
{ offset: 0, color: '#F4511E' },
{ offset: 0.25, color: '#FB8C00' },
{ offset: 0.5, color: '#FFB300' },
{ offset: 0.75, color: '#FDD835' },
{ offset: 1, color: '#7CB342' }
new graphic.LinearGradient(0, 0, 0, 1, [
{ offset: 0, color: '#FDD835' },
{ offset: 1, color: '#FB8C00' },
]),
new graphic.LinearGradient(0, 0, 0, 1, [
{ offset: 0, color: '#C0CA33' },
{ offset: 1, color: '#1B5E20' },
]),
],
grid: {
@ -126,33 +146,55 @@ export class BlockRewardsGraphComponent implements OnInit {
align: 'left',
},
borderColor: '#000',
formatter: (ticks) => {
let tooltip = `<b style="color: white; margin-left: 2px">${formatterXAxis(this.locale, this.timespan, parseInt(ticks[0].axisValue, 10))}</b><br>`;
tooltip += `${ticks[0].marker} ${ticks[0].seriesName}: ${formatNumber(ticks[0].data[1], this.locale, '1.3-3')} BTC`;
tooltip += `<br>`;
formatter: function (data) {
if (data.length <= 0) {
return '';
}
let tooltip = `<b style="color: white; margin-left: 2px">
${formatterXAxis(this.locale, this.timespan, parseInt(data[0].axisValue, 10))}</b><br>`;
if (['24h', '3d'].includes(this.timespan)) {
tooltip += `<small>` + $localize`At block: ${ticks[0].data[2]}` + `</small>`;
} else {
tooltip += `<small>` + $localize`Around block: ${ticks[0].data[2]}` + `</small>`;
for (const tick of data) {
if (tick.seriesIndex === 0) {
tooltip += `${tick.marker} ${tick.seriesName}: ${formatNumber(tick.data[1], this.locale, '1.3-3')} BTC<br>`;
} else if (tick.seriesIndex === 1) {
tooltip += `${tick.marker} ${tick.seriesName}: ${formatCurrency(tick.data[1], this.locale, getCurrencySymbol('USD', 'narrow'), 'USD', '1.0-0')}<br>`;
}
}
tooltip += `<small>* On average around block ${data[0].data[2]}</small>`;
return tooltip;
}
}.bind(this)
},
xAxis: {
name: formatterXAxisLabel(this.locale, this.timespan),
nameLocation: 'middle',
nameTextStyle: {
padding: [10, 0, 0, 0],
},
xAxis: data.blockRewards.length === 0 ? undefined :
{
type: 'time',
splitNumber: this.isMobile() ? 5 : 10,
axisLabel: {
hideOverlap: true,
}
},
yAxis: [
legend: data.blockRewards.length === 0 ? undefined : {
data: [
{
name: 'Rewards BTC',
inactiveColor: 'rgb(110, 112, 121)',
textStyle: {
color: 'white',
},
icon: 'roundRect',
},
{
name: 'Rewards USD',
inactiveColor: 'rgb(110, 112, 121)',
textStyle: {
color: 'white',
},
icon: 'roundRect',
},
],
},
yAxis: data.blockRewards.length === 0 ? undefined : [
{
min: value => Math.round(10 * value.min * 0.99) / 10,
max: value => Math.round(10 * value.max * 1.01) / 10,
type: 'value',
axisLabel: {
color: 'rgb(110, 112, 121)',
@ -160,6 +202,12 @@ export class BlockRewardsGraphComponent implements OnInit {
return `${val} BTC`;
}
},
min: (value) => {
return Math.round(value.min * (1.0 - scaleFactor) * 10) / 10;
},
max: (value) => {
return Math.round(value.max * (1.0 + scaleFactor) * 10) / 10;
},
splitLine: {
lineStyle: {
type: 'dotted',
@ -168,21 +216,56 @@ export class BlockRewardsGraphComponent implements OnInit {
}
},
},
],
series: [
{
zlevel: 0,
name: $localize`:@@12f86e6747a5ad39e62d3480ddc472b1aeab5b76:Reward`,
showSymbol: false,
symbol: 'none',
data: data.blockRewards,
type: 'line',
lineStyle: {
width: 2,
min: (value) => {
return Math.round(value.min * (1.0 - scaleFactor) * 10) / 10;
},
max: (value) => {
return Math.round(value.max * (1.0 + scaleFactor) * 10) / 10;
},
type: 'value',
position: 'right',
axisLabel: {
color: 'rgb(110, 112, 121)',
formatter: function(val) {
return this.fiatShortenerPipe.transform(val);
}.bind(this)
},
splitLine: {
show: false,
},
},
],
dataZoom: [{
series: data.blockRewards.length === 0 ? undefined : [
{
legendHoverLink: false,
zlevel: 0,
yAxisIndex: 0,
name: 'Rewards BTC',
data: data.blockRewards,
type: 'line',
smooth: 0.25,
symbol: 'none',
},
{
legendHoverLink: false,
zlevel: 1,
yAxisIndex: 1,
name: 'Rewards USD',
data: data.blockRewardsUSD,
type: 'line',
smooth: 0.25,
symbol: 'none',
lineStyle: {
width: 2,
opacity: 0.75,
},
areaStyle: {
opacity: 0.05,
}
},
],
dataZoom: data.blockRewards.length === 0 ? undefined : [{
type: 'inside',
realtime: true,
zoomLock: true,

View file

@ -351,6 +351,7 @@ export class HashrateChartComponent implements OnInit {
series: data.hashrates.length === 0 ? [] : [
{
zlevel: 0,
yAxisIndex: 0,
name: $localize`:@@79a9dc5b1caca3cbeb1733a19515edacc5fc7920:Hashrate`,
showSymbol: false,
symbol: 'none',

View file

@ -0,0 +1,37 @@
import { formatCurrency, getCurrencySymbol } from '@angular/common';
import { Inject, LOCALE_ID, Pipe, PipeTransform } from '@angular/core';
@Pipe({
name: 'fiatShortener'
})
export class FiatShortenerPipe implements PipeTransform {
constructor(
@Inject(LOCALE_ID) public locale: string
) {}
transform(num: number, ...args: any[]): unknown {
const digits = args[0] || 1;
const unit = args[1] || undefined;
if (num < 1000) {
return num.toFixed(digits);
}
const lookup = [
{ value: 1, symbol: '' },
{ value: 1e3, symbol: 'k' },
{ value: 1e6, symbol: 'M' },
{ value: 1e9, symbol: 'G' },
{ value: 1e12, symbol: 'T' },
{ value: 1e15, symbol: 'P' },
{ value: 1e18, symbol: 'E' }
];
const rx = /\.0+$|(\.[0-9]*[1-9])0+$/;
const item = lookup.slice().reverse().find((item) => num >= item.value);
let result = item ? (num / item.value).toFixed(digits).replace(rx, '$1') : '0';
result = formatCurrency(parseInt(result, 10), this.locale, getCurrencySymbol('USD', 'narrow'), 'USD', '1.0-0');
return result + item.symbol;
}
}