From 1969f2a275e59183348200fb4bc78af7bc02d4ff Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 7 Apr 2022 14:37:16 +0900 Subject: [PATCH 1/4] Use github api to fetch and update the pools database, once a week --- backend/src/api/pools-parser.ts | 14 +-- backend/src/index.ts | 4 +- backend/src/tasks/pools-updater.ts | 139 +++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 15 deletions(-) create mode 100644 backend/src/tasks/pools-updater.ts diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 005806c1d..dee95912a 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -17,23 +17,11 @@ class PoolsParser { /** * Parse the pools.json file, consolidate the data and dump it into the database */ - public async migratePoolsJson() { + public async migratePoolsJson(poolsJson: object) { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { return; } - logger.debug('Importing pools.json to the database, open ./pools.json'); - - let poolsJson: object = {}; - try { - const fileContent: string = readFileSync('./pools.json', 'utf8'); - poolsJson = JSON.parse(fileContent); - } catch (e) { - logger.err('Unable to open ./pools.json, does the file exist?'); - await this.insertUnknownPool(); - return; - } - // First we save every entries without paying attention to pool duplication const poolsDuplicated: Pool[] = []; diff --git a/backend/src/index.ts b/backend/src/index.ts index 008d987eb..20f6fb69c 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -22,12 +22,12 @@ import loadingIndicators from './api/loading-indicators'; import mempool from './api/mempool'; import elementsParser from './api/liquid/elements-parser'; import databaseMigration from './api/database-migration'; -import poolsParser from './api/pools-parser'; import syncAssets from './sync-assets'; import icons from './api/liquid/icons'; import { Common } from './api/common'; import mining from './api/mining'; import HashratesRepository from './repositories/HashratesRepository'; +import poolsUpdater from './tasks/pools-updater'; class Server { private wss: WebSocket.Server | undefined; @@ -99,7 +99,6 @@ class Server { await databaseMigration.$initializeOrMigrateDatabase(); if (Common.indexingEnabled()) { await this.$resetHashratesIndexingState(); - await poolsParser.migratePoolsJson(); } } catch (e) { throw new Error(e instanceof Error ? e.message : 'Error'); @@ -179,6 +178,7 @@ class Server { } try { + await poolsUpdater.updatePoolsJson(); blocks.$generateBlockDatabase(); await mining.$generateNetworkHashrateHistory(); await mining.$generatePoolHashrateHistory(); diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts new file mode 100644 index 000000000..a70e8cb5d --- /dev/null +++ b/backend/src/tasks/pools-updater.ts @@ -0,0 +1,139 @@ +const https = require('https'); +import poolsParser from "../api/pools-parser"; +import config from "../config"; +import { DB } from "../database"; +import logger from "../logger"; + +/** + * Maintain the most recent version of pools.json + */ +class PoolsUpdater { + lastRun: number = 0; + + constructor() { + } + + public async updatePoolsJson() { + if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { + return; + } + + const now = new Date().getTime() / 1000; + if (now - this.lastRun < 604800) { // Execute the PoolsUpdate only once a week, or upon restart + return; + } + + this.lastRun = now; + + try { + const dbSha = await this.getShaFromDb(); + const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github + if (githubSha === undefined) { + return; + } + + logger.debug(`Pools.json sha | Current: ${dbSha} | Github: ${githubSha}`); + if (dbSha !== undefined && dbSha === githubSha) { + return; + } + + logger.warn('Pools.json is outdated, fetch latest from github'); + const poolsJson = await this.fetchPools(); + await poolsParser.migratePoolsJson(poolsJson); + await this.updateDBSha(githubSha); + logger.notice('PoolsUpdater completed'); + + } catch (e) { + logger.err('PoolsUpdater failed. Error: ' + e); + } + } + + /** + * Fetch pools.json from github repo + */ + private async fetchPools(): Promise { + const response = await this.query('/repos/mempool/mining-pools/contents/pools.json'); + return JSON.parse(Buffer.from(response['content'], 'base64').toString('utf8')); + } + + /** + * Fetch our latest pools.json sha from the db + */ + private async updateDBSha(githubSha: string) { + let connection; + try { + connection = await DB.getConnection(); + await connection.query('DELETE FROM state where name="pools_json_sha"'); + await connection.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`); + connection.release(); + } catch (e) { + logger.err('Unable save github pools.json sha into the DB, error: ' + e); + connection.release(); + return undefined; + } + } + + /** + * Fetch our latest pools.json sha from the db + */ + private async getShaFromDb(): Promise { + let connection; + try { + connection = await DB.getConnection(); + const [rows] = await connection.query('SELECT string FROM state WHERE name="pools_json_sha"'); + connection.release(); + return (rows.length > 0 ? rows[0].string : undefined); + } catch (e) { + logger.err('Unable fetch pools.json sha from DB, error: ' + e); + connection.release(); + return undefined; + } + } + + /** + * Fetch our latest pools.json sha from github + */ + private async fetchPoolsSha(): Promise { + const response = await this.query('/repos/mempool/mining-pools/git/trees/master'); + + for (const file of response['tree']) { + if (file['path'] === 'pools.json') { + return file['sha']; + } + } + + logger.err('Unable to find latest pools.json sha from github'); + return undefined; + } + + /** + * Http request wrapper + */ + private async query(path): Promise { + return new Promise((resolve, reject) => { + const options = { + host: 'api.github.com', + path: path, + method: 'GET', + headers: { 'user-agent': 'node.js' } + }; + + logger.debug('Querying: api.github.com' + path); + + https.get(options, (response) => { + const chunks_of_data: any[] = []; + response.on('data', (fragments) => { + chunks_of_data.push(fragments); + }); + response.on('end', () => { + resolve(JSON.parse(Buffer.concat(chunks_of_data).toString())); + }); + response.on('error', (error) => { + reject(error); + }); + }); + }); + } +} + +export default new PoolsUpdater(); From 2d29b9ef89ef8a989898b86f90ff1cbb1b7ea8e8 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 7 Apr 2022 14:51:23 +0900 Subject: [PATCH 2/4] Upon error, re-run the PoolsUpdater within 24h instead of 7d --- backend/src/tasks/pools-updater.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index a70e8cb5d..e6883ed07 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -18,8 +18,11 @@ class PoolsUpdater { return; } + const oneWeek = 604800; + const oneDay = 86400; + const now = new Date().getTime() / 1000; - if (now - this.lastRun < 604800) { // Execute the PoolsUpdate only once a week, or upon restart + if (now - this.lastRun < oneWeek) { // Execute the PoolsUpdate only once a week, or upon restart return; } @@ -44,7 +47,8 @@ class PoolsUpdater { logger.notice('PoolsUpdater completed'); } catch (e) { - logger.err('PoolsUpdater failed. Error: ' + e); + this.lastRun = now - oneWeek - oneDay; // Try again in 24h + logger.err('PoolsUpdater failed. Will try again in 24h. Error: ' + e); } } From e451b40084b2527a534bbe9af21f7e31d4ed1d34 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 7 Apr 2022 16:14:43 +0900 Subject: [PATCH 3/4] Catch http request error - Fix 24h retry period --- backend/src/tasks/pools-updater.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index e6883ed07..b3838244a 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -47,7 +47,7 @@ class PoolsUpdater { logger.notice('PoolsUpdater completed'); } catch (e) { - this.lastRun = now - oneWeek - oneDay; // Try again in 24h + this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week logger.err('PoolsUpdater failed. Will try again in 24h. Error: ' + e); } } @@ -113,7 +113,7 @@ class PoolsUpdater { /** * Http request wrapper */ - private async query(path): Promise { + private query(path): Promise { return new Promise((resolve, reject) => { const options = { host: 'api.github.com', @@ -124,7 +124,7 @@ class PoolsUpdater { logger.debug('Querying: api.github.com' + path); - https.get(options, (response) => { + const request = https.get(options, (response) => { const chunks_of_data: any[] = []; response.on('data', (fragments) => { chunks_of_data.push(fragments); @@ -136,6 +136,11 @@ class PoolsUpdater { reject(error); }); }); + + request.on('error', (error) => { + logger.err('Query failed with error: ' + error); + reject(error); + }) }); } } From ba12b10f9d520acbee7458e7321a1161e97c416e Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 7 Apr 2022 18:14:28 +0900 Subject: [PATCH 4/4] Handle empty pools table error --- backend/src/api/blocks.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 80e7a4e1f..b1cfca8bd 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -135,6 +135,12 @@ class Blocks { } else { pool = await poolsRepository.$getUnknownPool(); } + + if (!pool) { // Something is wrong with the pools table, ignore pool indexing + logger.err('Unable to find pool, nor getting the unknown pool. Is the "pools" table empty?'); + return blockExtended; + } + blockExtended.extras.pool = { id: pool.id, name: pool.name,