From 53bc80e899fa45102cb9de896073b7a4165f3249 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 7 Jun 2022 11:28:39 +0200 Subject: [PATCH 1/2] Add 'db-less' mining pool tagging support --- backend/src/api/blocks.ts | 26 +++++++++++--- backend/src/api/pools-parser.ts | 57 +++++++++++++++++++++--------- backend/src/database.ts | 8 +++++ backend/src/indexer.ts | 4 ++- backend/src/tasks/pools-updater.ts | 26 ++++++++------ 5 files changed, 89 insertions(+), 32 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index a87d34a1a..1ec8531e4 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -18,6 +18,7 @@ import HashratesRepository from '../repositories/HashratesRepository'; import indexer from '../indexer'; import fiatConversion from './fiat-conversion'; import RatesRepository from '../repositories/RatesRepository'; +import poolsParser from './pools-parser'; class Blocks { private blocks: BlockExtended[] = []; @@ -139,7 +140,11 @@ class Blocks { if (blockExtended.extras?.coinbaseTx !== undefined) { pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx); } else { - pool = await poolsRepository.$getUnknownPool(); + if (config.DATABASE.ENABLED === true) { + pool = await poolsRepository.$getUnknownPool(); + } else { + pool = poolsParser.unknownPool; + } } if (!pool) { // We should never have this situation in practise @@ -165,13 +170,22 @@ class Blocks { */ private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined): Promise { if (txMinerInfo === undefined || txMinerInfo.vout.length < 1) { - return await poolsRepository.$getUnknownPool(); + if (config.DATABASE.ENABLED === true) { + return await poolsRepository.$getUnknownPool(); + } else { + return poolsParser.unknownPool; + } } const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig); const address = txMinerInfo.vout[0].scriptpubkey_address; - const pools: PoolTag[] = await poolsRepository.$getPools(); + let pools: PoolTag[] = []; + if (config.DATABASE.ENABLED === true) { + pools = await poolsRepository.$getPools(); + } else { + pools = poolsParser.miningPools; + } for (let i = 0; i < pools.length; ++i) { if (address !== undefined) { const addresses: string[] = JSON.parse(pools[i].addresses); @@ -190,7 +204,11 @@ class Blocks { } } - return await poolsRepository.$getUnknownPool(); + if (config.DATABASE.ENABLED === true) { + return await poolsRepository.$getUnknownPool(); + } else { + return poolsParser.unknownPool; + } } /** diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 202fca1bd..5d2c17227 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -11,6 +11,14 @@ interface Pool { } class PoolsParser { + miningPools: any[] = []; + unknownPool: any = { + 'name': "Unknown", + 'link': "https://learnmeabitcoin.com/technical/coinbase-transaction", + 'regexes': "[]", + 'addresses': "[]", + 'slug': 'unknown' + }; slugWarnFlag = false; /** @@ -60,12 +68,18 @@ class PoolsParser { // Get existing pools from the db let existingPools; try { - [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); + if (config.DATABASE.ENABLED === true) { + [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); + } else { + existingPools = []; + } } catch (e) { logger.err('Cannot get existing pools from the database, skipping pools.json import'); return; } + this.miningPools = []; + // Finally, we generate the final consolidated pools data const finalPoolDataAdd: Pool[] = []; const finalPoolDataUpdate: Pool[] = []; @@ -97,24 +111,33 @@ class PoolsParser { logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`); } + const poolObj = { + 'name': finalPoolName, + 'link': match[0].link, + 'regexes': allRegexes, + 'addresses': allAddresses, + 'slug': slug + }; + if (existingPools.find((pool) => pool.name === poolNames[i]) !== undefined) { - finalPoolDataUpdate.push({ - 'name': finalPoolName, - 'link': match[0].link, - 'regexes': allRegexes, - 'addresses': allAddresses, - 'slug': slug - }); + finalPoolDataUpdate.push(poolObj); } else { logger.debug(`Add '${finalPoolName}' mining pool`); - finalPoolDataAdd.push({ - 'name': finalPoolName, - 'link': match[0].link, - 'regexes': allRegexes, - 'addresses': allAddresses, - 'slug': slug - }); + finalPoolDataAdd.push(poolObj); } + + this.miningPools.push({ + 'name': finalPoolName, + 'link': match[0].link, + 'regexes': JSON.stringify(allRegexes), + 'addresses': JSON.stringify(allAddresses), + 'slug': slug + }); + } + + if (config.DATABASE.ENABLED === false) { // Don't run db operations + logger.info('Mining pools.json import completed (no database)'); + return; } logger.debug(`Update pools table now`); @@ -124,11 +147,11 @@ class PoolsParser { for (let i = 0; i < finalPoolDataAdd.length; ++i) { queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}', '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}', - ${JSON.stringify(finalPoolDataAdd[i].slug)}),`; + ${finalPoolDataAdd[i].slug}),`; } queryAdd = queryAdd.slice(0, -1) + ';'; - // Add new mining pools into the database + // Updated existing mining pools in the database const updateQueries: string[] = []; for (let i = 0; i < finalPoolDataUpdate.length; ++i) { updateQueries.push(` diff --git a/backend/src/database.ts b/backend/src/database.ts index 3816154cd..66c876378 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -22,12 +22,20 @@ import { PoolOptions } from 'mysql2/typings/mysql'; timezone: '+00:00', }; + private checkDBFlag() { + if (config.DATABASE.ENABLED === false) { + logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue'); + } + } + public async query(query, params?) { + this.checkDBFlag(); const pool = await this.getPool(); return pool.query(query, params); } public async checkDbConnection() { + this.checkDBFlag(); try { await this.query('SELECT ?', [1]); logger.info('Database connection established.'); diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 7ddc2a47f..04c51ffe2 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -13,7 +13,9 @@ class Indexer { } public reindex() { - this.runIndexer = true; + if (Common.indexingEnabled()) { + this.runIndexer = true; + } } public async $run() { diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 05a1da5dc..87487b17e 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -11,12 +11,13 @@ import * as https from 'https'; */ class PoolsUpdater { lastRun: number = 0; + currentSha: any = undefined; constructor() { } public async updatePoolsJson() { - if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || config.DATABASE.ENABLED === false) { + if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { return; } @@ -38,14 +39,17 @@ class PoolsUpdater { } try { - const dbSha = await this.getShaFromDb(); const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github if (githubSha === undefined) { return; } - logger.debug(`Pools.json sha | Current: ${dbSha} | Github: ${githubSha}`); - if (dbSha !== undefined && dbSha === githubSha) { + if (config.DATABASE.ENABLED === true) { + this.currentSha = await this.getShaFromDb(); + } + + logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`); + if (this.currentSha !== undefined && this.currentSha === githubSha) { return; } @@ -68,12 +72,14 @@ class PoolsUpdater { * Fetch our latest pools.json sha from the db */ private async updateDBSha(githubSha: string) { - try { - await DB.query('DELETE FROM state where name="pools_json_sha"'); - await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`); - } catch (e) { - logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e)); - return undefined; + this.currentSha = githubSha; + if (config.DATABASE.ENABLED === true) { + try { + await DB.query('DELETE FROM state where name="pools_json_sha"'); + await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`); + } catch (e) { + logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e)); + } } } From 21ae1fce2a988f1f54a4ea9a4c6e229298679e81 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 7 Jun 2022 12:19:36 +0200 Subject: [PATCH 2/2] Fix js crash when sending invalid state to /block page --- .../src/app/components/blocks-list/blocks-list.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.html b/frontend/src/app/components/blocks-list/blocks-list.component.html index 4a6bcf14a..dfaba73b2 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.html +++ b/frontend/src/app/components/blocks-list/blocks-list.component.html @@ -22,7 +22,7 @@ - {{ block.height }} + {{ block.height }}