From 63939ddbe41f6b1954e267c7161ad825a72dad79 Mon Sep 17 00:00:00 2001 From: Antoni Spaanderman <56turtle56@gmail.com> Date: Tue, 18 Jan 2022 22:25:38 +0100 Subject: [PATCH 01/52] outputs of genesis coinbase are always unspent --- backend/src/api/bitcoin/bitcoin-api.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 79505d5c3..c23e86ee0 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -106,10 +106,16 @@ class BitcoinApi implements AbstractBitcoinApi { const outSpends: IEsploraApi.Outspend[] = []; const tx = await this.$getRawTransaction(txId, true, false); for (let i = 0; i < tx.vout.length; i++) { - const txOut = await this.bitcoindClient.getTxOut(txId, i); - outSpends.push({ - spent: txOut === null, - }); + if (tx.status && tx.status.block_height == 0) { + outSpends.push({ + spent: false + }); + } else { + const txOut = await this.bitcoindClient.getTxOut(txId, i); + outSpends.push({ + spent: txOut === null, + }); + } } return outSpends; } From 2848f56c2b975c39533294a6f7309224e4c3e1c2 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 19 Jan 2022 18:50:52 +0900 Subject: [PATCH 02/52] Import mining pools into the database - Increment db schema to 3 --- backend/src/api/database-migration.ts | 20 ++++- backend/src/api/pools-parser.ts | 118 ++++++++++++++++++++++++++ backend/src/index.ts | 2 + 3 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 backend/src/api/pools-parser.ts diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 2ac97636e..9fdd4d210 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -3,10 +3,10 @@ import config from '../config'; import { DB } from '../database'; import logger from '../logger'; -const sleep = (ms: number) => new Promise( res => setTimeout(res, ms)); +const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 2; + private static currentVersion = 3; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -83,6 +83,9 @@ class DatabaseMigration { if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) { await this.$executeQuery(connection, `CREATE INDEX added ON statistics (added);`); } + if (databaseSchemaVersion < 3) { + await this.$executeQuery(connection, this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools')); + } connection.release(); } catch (e) { connection.release(); @@ -335,6 +338,17 @@ class DatabaseMigration { final_tx int(11) NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + + private getCreatePoolsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS pools ( + id int(11) NOT NULL AUTO_INCREMENT, + name varchar(50) NOT NULL, + link varchar(255) NOT NULL, + addresses text NOT NULL, + regexes text NOT NULL, + PRIMARY KEY (id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;`; + } } -export default new DatabaseMigration(); +export default new DatabaseMigration(); \ No newline at end of file diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts new file mode 100644 index 000000000..c17336288 --- /dev/null +++ b/backend/src/api/pools-parser.ts @@ -0,0 +1,118 @@ +import {readFileSync} from 'fs'; +import { DB } from '../database'; +import logger from '../logger'; + +interface Pool { + name: string, + link: string, + regexes: string[], + addresses: string[], +} + +class PoolsParser { + /** + * Parse the pools.json file, consolidate the data and dump it into the database + */ + public async migratePoolsJson() { + logger.info('Importing pools.json to the database'); + let connection = await DB.pool.getConnection(); + + // Check if the pools table does not have data already, for now we do not support updating it + // but that will come in a later version + let [rows] = await connection.query({ sql: 'SELECT count(id) as count from pools;', timeout: 120000 }); + if (rows[0].count !== 0) { + logger.info('Pools table already contain data, updating it is not yet supported, skipping.'); + connection.release(); + return; + } + + logger.info('Open ../frontend/cypress/fixtures/pools.json'); + const fileContent: string = readFileSync('../frontend/cypress/fixtures/pools.json','utf8'); + const poolsJson: object = JSON.parse(fileContent); + + // First we save every entries without paying attention to pool duplication + let poolsDuplicated: Pool[] = []; + + logger.info('Parse coinbase_tags'); + const coinbaseTags = Object.entries(poolsJson['coinbase_tags']); + for (let i = 0; i < coinbaseTags.length; ++i) { + poolsDuplicated.push({ + 'name': (coinbaseTags[i][1]).name, + 'link': (coinbaseTags[i][1]).link, + 'regexes': [coinbaseTags[i][0]], + 'addresses': [], + }); + } + logger.info('Parse payout_addresses'); + const addressesTags = Object.entries(poolsJson['payout_addresses']); + for (let i = 0; i < addressesTags.length; ++i) { + poolsDuplicated.push({ + 'name': (addressesTags[i][1]).name, + 'link': (addressesTags[i][1]).link, + 'regexes': [], + 'addresses': [addressesTags[i][0]], + }); + } + + // Then, we find unique mining pool names + logger.info('Identify unique mining pools'); + let poolNames : string[] = []; + for (let i = 0; i < poolsDuplicated.length; ++i) { + if (poolNames.indexOf(poolsDuplicated[i].name) === -1) { + poolNames.push(poolsDuplicated[i].name); + } + } + logger.info(`Found ${poolNames.length} unique mining pools`); + + // Finally, we generate the final consolidated pools data + let finalPoolData: Pool[] = []; + for (let i = 0; i < poolNames.length; ++i) { + let allAddresses: string[] = []; + let allRegexes: string[] = []; + let match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]); + + for (let y = 0; y < match.length; ++y) { + allAddresses = allAddresses.concat(match[y].addresses); + allRegexes = allRegexes.concat(match[y].regexes); + } + + finalPoolData.push({ + 'name': poolNames[i].replace("'", "''"), + 'link': match[0].link, + 'regexes': allRegexes, + 'addresses': allAddresses, + }) + } + + // Manually add the 'unknown pool' + finalPoolData.push({ + 'name': 'Unknown', + 'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction', + regexes: [], + addresses: [], + }) + + // Dump everything into the database + logger.info(`Insert mining pool info into the database`); + let query: string = 'INSERT INTO pools(name, link, regexes, addresses) VALUES '; + for (let i = 0; i < finalPoolData.length; ++i) { + query += `('${finalPoolData[i].name}', '${finalPoolData[i].link}', + '${JSON.stringify(finalPoolData[i].regexes)}', '${JSON.stringify(finalPoolData[i].addresses)}'),`; + } + query = query.slice(0, -1) + ';'; + + try { + await connection.query({ sql: 'DELETE FROM pools;', timeout: 120000 }); // We clear the table before insertion + await connection.query({ sql: query, timeout: 120000 }); + connection.release(); + logger.info('Import completed'); + } catch (e) { + connection.release(); + logger.info(`Unable to import pools in the database!`); + throw e; + } + } + +} + +export default new PoolsParser(); \ No newline at end of file diff --git a/backend/src/index.ts b/backend/src/index.ts index f6615d1c8..9e4dcee35 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -22,6 +22,7 @@ import loadingIndicators from './api/loading-indicators'; import mempool from './api/mempool'; import elementsParser from './api/liquid/elements-parser'; import databaseMigration from './api/database-migration'; +import poolsParser from './api/pools-parser'; import syncAssets from './sync-assets'; import icons from './api/liquid/icons'; import { Common } from './api/common'; @@ -88,6 +89,7 @@ class Server { await checkDbConnection(); try { await databaseMigration.$initializeOrMigrateDatabase(); + await poolsParser.migratePoolsJson(); } catch (e) { throw new Error(e instanceof Error ? e.message : 'Error'); } From 979c52d3c413532411e0ca713333f12592e21c69 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 20 Jan 2022 13:53:08 +0900 Subject: [PATCH 03/52] Add pools.json to EXTERNAL_ASSETS - Now supports updating the table --- backend/mempool-config.sample.json | 4 +- backend/src/api/pools-parser.ts | 94 +++++++++++++++++++----------- 2 files changed, 63 insertions(+), 35 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index ea656c1de..1b55f38f4 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -14,7 +14,9 @@ "MEMPOOL_BLOCKS_AMOUNT": 8, "PRICE_FEED_UPDATE_INTERVAL": 3600, "USE_SECOND_NODE_FOR_MINFEE": false, - "EXTERNAL_ASSETS": [] + "EXTERNAL_ASSETS": [ + "https://mempool.space/resources/pools.json" + ] }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index c17336288..5e6a38e49 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -14,20 +14,14 @@ class PoolsParser { * Parse the pools.json file, consolidate the data and dump it into the database */ public async migratePoolsJson() { + const connection = await DB.pool.getConnection(); logger.info('Importing pools.json to the database'); - let connection = await DB.pool.getConnection(); - // Check if the pools table does not have data already, for now we do not support updating it - // but that will come in a later version - let [rows] = await connection.query({ sql: 'SELECT count(id) as count from pools;', timeout: 120000 }); - if (rows[0].count !== 0) { - logger.info('Pools table already contain data, updating it is not yet supported, skipping.'); - connection.release(); - return; - } + // Get existing pools from the db + const [existingPools] = await connection.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); // We clear the table before insertion - logger.info('Open ../frontend/cypress/fixtures/pools.json'); - const fileContent: string = readFileSync('../frontend/cypress/fixtures/pools.json','utf8'); + logger.info('Open ./pools.json'); + const fileContent: string = readFileSync('./pools.json','utf8'); const poolsJson: object = JSON.parse(fileContent); // First we save every entries without paying attention to pool duplication @@ -65,7 +59,8 @@ class PoolsParser { logger.info(`Found ${poolNames.length} unique mining pools`); // Finally, we generate the final consolidated pools data - let finalPoolData: Pool[] = []; + let finalPoolDataAdd: Pool[] = []; + let finalPoolDataUpdate: Pool[] = []; for (let i = 0; i < poolNames.length; ++i) { let allAddresses: string[] = []; let allRegexes: string[] = []; @@ -76,34 +71,65 @@ class PoolsParser { allRegexes = allRegexes.concat(match[y].regexes); } - finalPoolData.push({ - 'name': poolNames[i].replace("'", "''"), - 'link': match[0].link, - 'regexes': allRegexes, - 'addresses': allAddresses, - }) + const finalPoolName = poolNames[i].replace("'", "''"); // To support single quote in names when doing db queries + + if (existingPools.find((pool) => { return pool.name === poolNames[i]}) !== undefined) { + logger.debug(`Update '${finalPoolName} mining pool`); + finalPoolDataUpdate.push({ + 'name': finalPoolName, + 'link': match[0].link, + 'regexes': allRegexes, + 'addresses': allAddresses, + }) + } else { + logger.debug(`Add '${finalPoolName} mining pool`); + finalPoolDataAdd.push({ + 'name': finalPoolName, + 'link': match[0].link, + 'regexes': allRegexes, + 'addresses': allAddresses, + }) + } } // Manually add the 'unknown pool' - finalPoolData.push({ - 'name': 'Unknown', - 'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction', - regexes: [], - addresses: [], - }) - - // Dump everything into the database - logger.info(`Insert mining pool info into the database`); - let query: string = 'INSERT INTO pools(name, link, regexes, addresses) VALUES '; - for (let i = 0; i < finalPoolData.length; ++i) { - query += `('${finalPoolData[i].name}', '${finalPoolData[i].link}', - '${JSON.stringify(finalPoolData[i].regexes)}', '${JSON.stringify(finalPoolData[i].addresses)}'),`; + if (existingPools.find((pool) => { return pool.name === "Uknown"}) !== undefined) { + finalPoolDataAdd.push({ + 'name': 'Unknown', + 'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction', + regexes: [], + addresses: [], + }) + } + + logger.info(`Update pools table now`); + + // Add new mining pools into the database + let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses) VALUES '; + for (let i = 0; i < finalPoolDataAdd.length; ++i) { + queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}', + '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}'),`; + } + queryAdd = queryAdd.slice(0, -1) + ';'; + + // Add new mining pools into the database + let updateQueries: string[] = []; + for (let i = 0; i < finalPoolDataUpdate.length; ++i) { + updateQueries.push(` + UPDATE pools + SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}', + regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}' + WHERE name='${finalPoolDataUpdate[i].name}' + ;`); } - query = query.slice(0, -1) + ';'; try { - await connection.query({ sql: 'DELETE FROM pools;', timeout: 120000 }); // We clear the table before insertion - await connection.query({ sql: query, timeout: 120000 }); + if (finalPoolDataAdd.length > 0) { + await connection.query({ sql: queryAdd, timeout: 120000 }); + } + updateQueries.forEach(async query => { + await connection.query({ sql: query, timeout: 120000 }); + }); connection.release(); logger.info('Import completed'); } catch (e) { From 1210643e8ec9aaaa6952f000bdf9c82bdc5dc0e1 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 20 Jan 2022 16:34:14 +0900 Subject: [PATCH 04/52] Fix linter issues and typo --- backend/src/api/pools-parser.ts | 42 ++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 5e6a38e49..da048f2d3 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -3,10 +3,10 @@ import { DB } from '../database'; import logger from '../logger'; interface Pool { - name: string, - link: string, - regexes: string[], - addresses: string[], + name: string; + link: string; + regexes: string[]; + addresses: string[]; } class PoolsParser { @@ -18,14 +18,14 @@ class PoolsParser { logger.info('Importing pools.json to the database'); // Get existing pools from the db - const [existingPools] = await connection.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); // We clear the table before insertion + const [existingPools] = await connection.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); logger.info('Open ./pools.json'); - const fileContent: string = readFileSync('./pools.json','utf8'); + const fileContent: string = readFileSync('./pools.json', 'utf8'); const poolsJson: object = JSON.parse(fileContent); // First we save every entries without paying attention to pool duplication - let poolsDuplicated: Pool[] = []; + const poolsDuplicated: Pool[] = []; logger.info('Parse coinbase_tags'); const coinbaseTags = Object.entries(poolsJson['coinbase_tags']); @@ -50,7 +50,7 @@ class PoolsParser { // Then, we find unique mining pool names logger.info('Identify unique mining pools'); - let poolNames : string[] = []; + const poolNames: string[] = []; for (let i = 0; i < poolsDuplicated.length; ++i) { if (poolNames.indexOf(poolsDuplicated[i].name) === -1) { poolNames.push(poolsDuplicated[i].name); @@ -59,47 +59,47 @@ class PoolsParser { logger.info(`Found ${poolNames.length} unique mining pools`); // Finally, we generate the final consolidated pools data - let finalPoolDataAdd: Pool[] = []; - let finalPoolDataUpdate: Pool[] = []; + const finalPoolDataAdd: Pool[] = []; + const finalPoolDataUpdate: Pool[] = []; for (let i = 0; i < poolNames.length; ++i) { let allAddresses: string[] = []; let allRegexes: string[] = []; - let match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]); + const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]); for (let y = 0; y < match.length; ++y) { allAddresses = allAddresses.concat(match[y].addresses); allRegexes = allRegexes.concat(match[y].regexes); } - const finalPoolName = poolNames[i].replace("'", "''"); // To support single quote in names when doing db queries + const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries - if (existingPools.find((pool) => { return pool.name === poolNames[i]}) !== undefined) { - logger.debug(`Update '${finalPoolName} mining pool`); + if (existingPools.find((pool) => pool.name === poolNames[i]) !== undefined) { + logger.debug(`Update '${finalPoolName}' mining pool`); finalPoolDataUpdate.push({ 'name': finalPoolName, 'link': match[0].link, 'regexes': allRegexes, 'addresses': allAddresses, - }) + }); } else { - logger.debug(`Add '${finalPoolName} mining pool`); + logger.debug(`Add '${finalPoolName}' mining pool`); finalPoolDataAdd.push({ 'name': finalPoolName, 'link': match[0].link, 'regexes': allRegexes, 'addresses': allAddresses, - }) + }); } } // Manually add the 'unknown pool' - if (existingPools.find((pool) => { return pool.name === "Uknown"}) !== undefined) { + if (existingPools.find((pool) => pool.name === 'Unknown') !== undefined) { finalPoolDataAdd.push({ 'name': 'Unknown', 'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction', regexes: [], addresses: [], - }) + }); } logger.info(`Update pools table now`); @@ -113,7 +113,7 @@ class PoolsParser { queryAdd = queryAdd.slice(0, -1) + ';'; // Add new mining pools into the database - let updateQueries: string[] = []; + const updateQueries: string[] = []; for (let i = 0; i < finalPoolDataUpdate.length; ++i) { updateQueries.push(` UPDATE pools @@ -141,4 +141,4 @@ class PoolsParser { } -export default new PoolsParser(); \ No newline at end of file +export default new PoolsParser(); From 8d1cc40459679d10f0ce12b44a3452fc2208d827 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 20 Jan 2022 16:56:25 +0900 Subject: [PATCH 05/52] Fix add 'Unknown' pool logic --- backend/src/api/pools-parser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index da048f2d3..9bfcd3366 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -93,7 +93,7 @@ class PoolsParser { } // Manually add the 'unknown pool' - if (existingPools.find((pool) => pool.name === 'Unknown') !== undefined) { + if (existingPools.find((pool) => pool.name === 'Unknown') === undefined) { finalPoolDataAdd.push({ 'name': 'Unknown', 'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction', From 19a564062bed4ced0a5e407a744218941ca5c77d Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 20 Jan 2022 22:59:10 +0900 Subject: [PATCH 06/52] Add pools.json file in default config.ts - Handle file exception - Only import pools for MAINNET --- backend/src/api/pools-parser.ts | 82 ++++++++++++++++++++------------- backend/src/config.ts | 4 +- 2 files changed, 54 insertions(+), 32 deletions(-) diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 9bfcd3366..bb7779089 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -1,6 +1,7 @@ -import {readFileSync} from 'fs'; +import { readFileSync } from 'fs'; import { DB } from '../database'; import logger from '../logger'; +import config from '../config'; interface Pool { name: string; @@ -14,20 +15,26 @@ class PoolsParser { * Parse the pools.json file, consolidate the data and dump it into the database */ public async migratePoolsJson() { - const connection = await DB.pool.getConnection(); - logger.info('Importing pools.json to the database'); + if (config.MEMPOOL.NETWORK !== 'mainnet') { + return; + } - // Get existing pools from the db - const [existingPools] = await connection.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); + logger.debug('Importing pools.json to the database, open ./pools.json'); - logger.info('Open ./pools.json'); - const fileContent: string = readFileSync('./pools.json', 'utf8'); - const poolsJson: object = JSON.parse(fileContent); + let poolsJson: object = {}; + try { + const fileContent: string = readFileSync('./pools.json', 'utf8'); + poolsJson = JSON.parse(fileContent); + } catch (e) { + logger.err('Unable to open ./pools.json, does the file exist?'); + await this.insertUnknownPool(); + return; + } // First we save every entries without paying attention to pool duplication const poolsDuplicated: Pool[] = []; - logger.info('Parse coinbase_tags'); + logger.debug('Parse coinbase_tags'); const coinbaseTags = Object.entries(poolsJson['coinbase_tags']); for (let i = 0; i < coinbaseTags.length; ++i) { poolsDuplicated.push({ @@ -37,7 +44,7 @@ class PoolsParser { 'addresses': [], }); } - logger.info('Parse payout_addresses'); + logger.debug('Parse payout_addresses'); const addressesTags = Object.entries(poolsJson['payout_addresses']); for (let i = 0; i < addressesTags.length; ++i) { poolsDuplicated.push({ @@ -49,14 +56,18 @@ class PoolsParser { } // Then, we find unique mining pool names - logger.info('Identify unique mining pools'); + logger.debug('Identify unique mining pools'); const poolNames: string[] = []; for (let i = 0; i < poolsDuplicated.length; ++i) { if (poolNames.indexOf(poolsDuplicated[i].name) === -1) { poolNames.push(poolsDuplicated[i].name); } } - logger.info(`Found ${poolNames.length} unique mining pools`); + logger.debug(`Found ${poolNames.length} unique mining pools`); + + // Get existing pools from the db + const connection = await DB.pool.getConnection(); + const [existingPools] = await connection.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); // Finally, we generate the final consolidated pools data const finalPoolDataAdd: Pool[] = []; @@ -92,23 +103,13 @@ class PoolsParser { } } - // Manually add the 'unknown pool' - if (existingPools.find((pool) => pool.name === 'Unknown') === undefined) { - finalPoolDataAdd.push({ - 'name': 'Unknown', - 'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction', - regexes: [], - addresses: [], - }); - } - - logger.info(`Update pools table now`); + logger.debug(`Update pools table now`); // Add new mining pools into the database let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses) VALUES '; for (let i = 0; i < finalPoolDataAdd.length; ++i) { queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}', - '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}'),`; + '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}'),`; } queryAdd = queryAdd.slice(0, -1) + ';'; @@ -116,11 +117,11 @@ class PoolsParser { const updateQueries: string[] = []; for (let i = 0; i < finalPoolDataUpdate.length; ++i) { updateQueries.push(` - UPDATE pools - SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}', - regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}' - WHERE name='${finalPoolDataUpdate[i].name}' - ;`); + UPDATE pools + SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}', + regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}' + WHERE name='${finalPoolDataUpdate[i].name}' + ;`); } try { @@ -130,15 +131,34 @@ class PoolsParser { updateQueries.forEach(async query => { await connection.query({ sql: query, timeout: 120000 }); }); + await this.insertUnknownPool(); connection.release(); - logger.info('Import completed'); + logger.info('Mining pools.json import completed'); } catch (e) { connection.release(); - logger.info(`Unable to import pools in the database!`); + logger.err(`Unable to import pools in the database!`); throw e; } } + /** + * Manually add the 'unknown pool' + */ + private async insertUnknownPool() { + const connection = await DB.pool.getConnection(); + try { + const [rows]: any[] = await connection.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 }); + if (rows.length === 0) { + logger.debug('Manually inserting "Unknown" mining pool into the databse'); + await connection.query({ + sql: `INSERT INTO pools(name, link, regexes, addresses) + VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]"); + `}); + } + } catch (e) { + logger.err('Unable to insert "Unknown" mining pool'); + } + } } export default new PoolsParser(); diff --git a/backend/src/config.ts b/backend/src/config.ts index 4c2888834..3cc928327 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -79,7 +79,9 @@ const defaults: IConfig = { 'MEMPOOL_BLOCKS_AMOUNT': 8, 'PRICE_FEED_UPDATE_INTERVAL': 3600, 'USE_SECOND_NODE_FOR_MINFEE': false, - 'EXTERNAL_ASSETS': [], + 'EXTERNAL_ASSETS': [ + 'https://mempool.space/resources/pools.json' + ] }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', From a1a2e9363fc7137980ba37a2e80889539a50cd4a Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 20 Jan 2022 23:07:20 +0900 Subject: [PATCH 07/52] Make sure to release all db connections --- backend/src/api/pools-parser.ts | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index bb7779089..2b0901551 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -67,7 +67,14 @@ class PoolsParser { // Get existing pools from the db const connection = await DB.pool.getConnection(); - const [existingPools] = await connection.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); + let existingPools: any[] = []; + try { + existingPools = await connection.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); + } catch (e) { + logger.err('Unable to get existing pools from the database, skipping pools.json import'); + connection.release(); + return; + } // Finally, we generate the final consolidated pools data const finalPoolDataAdd: Pool[] = []; @@ -117,11 +124,11 @@ class PoolsParser { const updateQueries: string[] = []; for (let i = 0; i < finalPoolDataUpdate.length; ++i) { updateQueries.push(` - UPDATE pools - SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}', - regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}' - WHERE name='${finalPoolDataUpdate[i].name}' - ;`); + UPDATE pools + SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}', + regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}' + WHERE name='${finalPoolDataUpdate[i].name}' + ;`); } try { @@ -158,6 +165,8 @@ class PoolsParser { } catch (e) { logger.err('Unable to insert "Unknown" mining pool'); } + + connection.release(); } } From 87175869dd741ca366eddce5bdcc36928157e552 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 20 Jan 2022 23:31:32 +0900 Subject: [PATCH 08/52] Fix typescript miss use --- backend/src/api/pools-parser.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 2b0901551..e3f5f0b89 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -67,9 +67,9 @@ class PoolsParser { // Get existing pools from the db const connection = await DB.pool.getConnection(); - let existingPools: any[] = []; + let existingPools; try { - existingPools = await connection.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); + [existingPools] = await connection.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); } catch (e) { logger.err('Unable to get existing pools from the database, skipping pools.json import'); connection.release(); From a8c04624f08f4550c2ae3b4e2c909936845ad7e6 Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 21 Jan 2022 01:32:19 +0400 Subject: [PATCH 09/52] Fixing liqud asset precision fixes #1166 --- .../transactions-list/transactions-list.component.html | 2 +- .../transactions-list/transactions-list.component.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.html b/frontend/src/app/components/transactions-list/transactions-list.component.html index 36aed7fe6..680b6be53 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.html +++ b/frontend/src/app/components/transactions-list/transactions-list.component.html @@ -270,7 +270,7 @@ - {{ item.value / 100000000 | number: '1.0-' + assetsMinimal[item.asset][3] }} {{ assetsMinimal[item.asset][1] }} + {{ item.value / pow(10, assetsMinimal[item.asset][3]) | number: '1.' + assetsMinimal[item.asset][3] + '-' + assetsMinimal[item.asset][3] }} {{ assetsMinimal[item.asset][1] }}
{{ assetsMinimal[item.asset][0] }}
diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.ts b/frontend/src/app/components/transactions-list/transactions-list.component.ts index ecddf4436..e1fb7d2e6 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.ts +++ b/frontend/src/app/components/transactions-list/transactions-list.component.ts @@ -119,6 +119,10 @@ export class TransactionsListComponent implements OnInit, OnChanges { return '0x' + (str.length % 2 ? '0' : '') + str; } + pow(base: number, exponent: number): number { + return Math.pow(base, exponent); + } + toggleDetails() { this.displayDetails = !this.displayDetails; this.ref.markForCheck(); From 056049615485831b27b4aec605566f37234b0751 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Fri, 21 Jan 2022 23:12:18 -0800 Subject: [PATCH 10/52] Fix broken link on the Bisq transaction page --- .../app/bisq/bisq-transaction/bisq-transaction.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.html b/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.html index 57862fed6..2b3964caa 100644 --- a/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.html +++ b/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.html @@ -7,7 +7,7 @@ - + {{ bisqTx.id | shortenString : 24 }} {{ bisqTx.id }} From a5ca0cda1427580bea1dc28c4f5750cf9e3f1d4e Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Sat, 22 Jan 2022 14:21:46 -0800 Subject: [PATCH 11/52] Add ids to nav bar items, liquid and bisq components --- .../bisq-transactions.component.html | 4 ++-- .../bisq-master-page.component.html | 12 +++++------ .../liquid-master-page.component.html | 12 +++++------ .../master-page/master-page.component.html | 20 +++++++++---------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/frontend/src/app/bisq/bisq-transactions/bisq-transactions.component.html b/frontend/src/app/bisq/bisq-transactions/bisq-transactions.component.html index f85c29a9b..d1064972e 100644 --- a/frontend/src/app/bisq/bisq-transactions/bisq-transactions.component.html +++ b/frontend/src/app/bisq/bisq-transactions/bisq-transactions.component.html @@ -1,7 +1,7 @@

BSQ Transactions

-
+
@@ -39,7 +39,7 @@ {{ tx.blockHeight }} - + diff --git a/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html b/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html index d77f96423..cacd38f39 100644 --- a/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html +++ b/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html @@ -27,22 +27,22 @@