mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2024-12-26 16:24:26 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1876 from mempool/nymkappa/bugfix/cleanup-log

Browse Source 
Cleanup some ops logs
This commit is contained in:
wiz 2022-06-14 08:29:21 +09:00 committed by GitHub
commit 19883c03ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 59 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/api/blocks.ts
View File

@ -280,7 +280,7 @@ class Blocks {
currentBlockHeight -= chunkSize; currentBlockHeight -= chunkSize;
} }
logger.info(`Indexed ${newlyIndexed} blocks`); logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`);
loadingIndicators.setProgress('block-indexing', 100); loadingIndicators.setProgress('block-indexing', 100);
} catch (e) { } catch (e) {
logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e)); logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));

50
backend/src/api/database-migration.ts
View File

@ -7,8 +7,24 @@ class DatabaseMigration {
private static currentVersion = 19; private static currentVersion = 19;
private queryTimeout = 120000; private queryTimeout = 120000;
private statisticsAddedIndexed = false; private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
private blocksTruncatedMessage = `'blocks' table has been truncated. Re-indexing from scratch.`;
private hashratesTruncatedMessage = `'hashrates' table has been truncated. Re-indexing from scratch.`;
constructor() { } constructor() { }
/**
* Avoid printing multiple time the same message
*/
private uniqueLog(loggerFunction: any, msg: string) {
if (this.uniqueLogs.includes(msg)) {
return;
}
this.uniqueLogs.push(msg);
loggerFunction(msg);
}
/** /**
* Entry point * Entry point
*/ */
@ -39,6 +55,16 @@ class DatabaseMigration {
process.exit(-1); process.exit(-1);
} }
if (databaseSchemaVersion === 0) {
logger.info('Initializing database (first run, clean install)');
}
if (databaseSchemaVersion <= 2) {
// Disable some spam logs when they're not relevant
this.uniqueLogs.push(this.blocksTruncatedMessage);
this.uniqueLogs.push(this.hashratesTruncatedMessage);
}
logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion); logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
logger.debug('MIGRATIONS: Latest DatabaseMigration.version is ' + DatabaseMigration.currentVersion); logger.debug('MIGRATIONS: Latest DatabaseMigration.version is ' + DatabaseMigration.currentVersion);
if (databaseSchemaVersion >= DatabaseMigration.currentVersion) { if (databaseSchemaVersion >= DatabaseMigration.currentVersion) {
@ -56,10 +82,13 @@ class DatabaseMigration {
} }
if (DatabaseMigration.currentVersion > databaseSchemaVersion) { if (DatabaseMigration.currentVersion > databaseSchemaVersion) {
logger.notice('MIGRATIONS: Upgrading database schema');
try { try {
await this.$migrateTableSchemaFromVersion(databaseSchemaVersion); await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
if (databaseSchemaVersion === 0) {
logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);
} else {
logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`); logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
}
} catch (e) { } catch (e) {
logger.err('MIGRATIONS: Unable to migrate database, aborting. ' + e); logger.err('MIGRATIONS: Unable to migrate database, aborting. ' + e);
} }
@ -89,13 +118,13 @@ class DatabaseMigration {
await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks')); await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
} }
if (databaseSchemaVersion < 5 && isBitcoin === true) { if (databaseSchemaVersion < 5 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`); this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"'); await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
} }
if (databaseSchemaVersion < 6 && isBitcoin === true) { if (databaseSchemaVersion < 6 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`); this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
// Cleanup original blocks fields type // Cleanup original blocks fields type
await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"'); await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
@ -122,7 +151,7 @@ class DatabaseMigration {
} }
if (databaseSchemaVersion < 8 && isBitcoin === true) { if (databaseSchemaVersion < 8 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`); this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`'); await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST'); await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
@ -131,7 +160,7 @@ class DatabaseMigration {
} }
if (databaseSchemaVersion < 9 && isBitcoin === true) { if (databaseSchemaVersion < 9 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`); this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)'); await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)'); await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
@ -142,7 +171,7 @@ class DatabaseMigration {
} }
if (databaseSchemaVersion < 11 && isBitcoin === true) { if (databaseSchemaVersion < 11 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`); this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery(`ALTER TABLE blocks await this.$executeQuery(`ALTER TABLE blocks
ADD avg_fee INT UNSIGNED NULL, ADD avg_fee INT UNSIGNED NULL,
@ -166,14 +195,14 @@ class DatabaseMigration {
} }
if (databaseSchemaVersion < 14 && isBitcoin === true) { if (databaseSchemaVersion < 14 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`); this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`'); await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"'); await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
} }
if (databaseSchemaVersion < 16 && isBitcoin === true) { if (databaseSchemaVersion < 16 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`); this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
} }
@ -282,6 +311,8 @@ class DatabaseMigration {
for (const query of this.getMigrationQueriesFromVersion(version)) { for (const query of this.getMigrationQueriesFromVersion(version)) {
transactionQueries.push(query); transactionQueries.push(query);
} }
logger.notice(`MIGRATIONS: ${version > 0 ? 'Upgrading' : 'Initializing'} database schema version number to ${DatabaseMigration.currentVersion}`);
transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery()); transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery());
try { try {
@ -305,6 +336,9 @@ class DatabaseMigration {
if (version < 1) { if (version < 1) {
if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') { if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') {
if (version > 0) {
logger.notice(`MIGRATIONS: Migrating (shifting) statistics table data`);
}
queries.push(this.getShiftStatisticsQuery()); queries.push(this.getShiftStatisticsQuery());
} }
} }

4
backend/src/api/mining.ts
View File

@ -257,7 +257,7 @@ class Mining {
} }
await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate()); await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
if (newlyIndexed > 0) { if (newlyIndexed > 0) {
logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`); logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`);
} }
loadingIndicators.setProgress('weekly-hashrate-indexing', 100); loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
} catch (e) { } catch (e) {
@ -368,7 +368,7 @@ class Mining {
await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate()); await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
if (newlyIndexed > 0) { if (newlyIndexed > 0) {
logger.info(`Indexed ${newlyIndexed} day of network hashrate`); logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`);
} }
loadingIndicators.setProgress('daily-hashrate-indexing', 100); loadingIndicators.setProgress('daily-hashrate-indexing', 100);
} catch (e) { } catch (e) {

21
backend/src/tasks/pools-updater.ts
View File

@ -13,6 +13,8 @@ import * as https from 'https';
class PoolsUpdater { class PoolsUpdater {
lastRun: number = 0; lastRun: number = 0;
currentSha: any = undefined; currentSha: any = undefined;
poolsUrl: string = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json';
treeUrl: string = 'https://api.github.com/repos/mempool/mining-pools/git/trees/master';
constructor() { constructor() {
} }
@ -32,11 +34,10 @@ class PoolsUpdater {
this.lastRun = now; this.lastRun = now;
logger.info('Updating latest mining pools from Github');
if (config.SOCKS5PROXY.ENABLED) { if (config.SOCKS5PROXY.ENABLED) {
logger.info('List of public pools will be queried over the Tor network'); logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`);
} else { } else {
logger.info('List of public pools will be queried over clearnet'); logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`);
} }
try { try {
@ -54,8 +55,12 @@ class PoolsUpdater {
return; return;
} }
logger.warn('Pools.json is outdated, fetch latest from github'); if (this.currentSha === undefined) {
const poolsJson = await this.query('https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'); logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`);
} else {
logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`);
}
const poolsJson = await this.query(this.poolsUrl);
if (poolsJson === undefined) { if (poolsJson === undefined) {
return; return;
} }
@ -101,7 +106,7 @@ class PoolsUpdater {
* Fetch our latest pools.json sha from github * Fetch our latest pools.json sha from github
*/ */
private async fetchPoolsSha(): Promise<string | undefined> { private async fetchPoolsSha(): Promise<string | undefined> {
const response = await this.query('https://api.github.com/repos/mempool/mining-pools/git/trees/master'); const response = await this.query(this.treeUrl);
if (response !== undefined) { if (response !== undefined) {
for (const file of response['tree']) { for (const file of response['tree']) {
@ -111,7 +116,7 @@ class PoolsUpdater {
} }
} }
logger.err('Cannot to find latest pools.json sha from github api response'); logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`);
return undefined; return undefined;
} }
@ -159,7 +164,7 @@ class PoolsUpdater {
const data: AxiosResponse = await axios.get(path, axiosOptions); const data: AxiosResponse = await axios.get(path, axiosOptions);
if (data.statusText === 'error' || !data.data) { if (data.statusText === 'error' || !data.data) {
throw new Error(`Could not fetch data from Github, Error: ${data.status}`); throw new Error(`Could not fetch data from ${path}, Error: ${data.status}`);
} }
return data.data; return data.data;
} catch (e) { } catch (e) {