From ec4c418c22339f511008d090fd57056dd0744f01 Mon Sep 17 00:00:00 2001 From: James Blacklock Date: Wed, 20 Dec 2023 17:02:12 -0500 Subject: [PATCH 01/40] sign contributor agreement --- contributors/jamesblacklock.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contributors/jamesblacklock.txt diff --git a/contributors/jamesblacklock.txt b/contributors/jamesblacklock.txt new file mode 100644 index 000000000..11591f451 --- /dev/null +++ b/contributors/jamesblacklock.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of December 20, 2023. + +Signed: jamesblacklock From cd964d37e806811fa52dba0647e70efd802a9050 Mon Sep 17 00:00:00 2001 From: James Blacklock Date: Wed, 20 Dec 2023 17:03:06 -0500 Subject: [PATCH 02/40] fix bug in backend Docker start.sh script --- backend/.gitignore | 6 ++++++ docker/backend/start.sh | 2 +- frontend/.gitignore | 7 +++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/backend/.gitignore b/backend/.gitignore index b4393c2f0..5cefd4bab 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -7,6 +7,12 @@ mempool-config.json pools.json icons.json +# docker +Dockerfile +GeoIP +start.sh +wait-for-it.sh + # compiled output /dist /tmp diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 232cf7284..aab5ad489 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -53,7 +53,7 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false} # ESPLORA __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000} -__ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:="null"} +__ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:=""} __ESPLORA_BATCH_QUERY_BASE_SIZE__=${ESPLORA_BATCH_QUERY_BASE_SIZE:=1000} __ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000} __ESPLORA_REQUEST_TIMEOUT__=${ESPLORA_REQUEST_TIMEOUT:=5000} diff --git a/frontend/.gitignore b/frontend/.gitignore index 8159e7c7b..d2a765dda 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -6,6 +6,13 @@ /out-tsc server.run.js +# docker +Dockerfile +entrypoint.sh +nginx-mempool.conf +nginx.conf +wait-for + # Only exists if Bazel was run /bazel-out From 4b10e32e73282c3bea7f0d928ea11ef033f7adf8 Mon Sep 17 00:00:00 2001 From: natsee Date: Sat, 20 Jan 2024 15:15:15 +0100 Subject: [PATCH 03/40] Liquid: add indexing process of Federation utxos --- backend/src/api/database-migration.ts | 43 ++- backend/src/api/liquid/elements-parser.ts | 305 +++++++++++++++++++++- backend/src/api/liquid/liquid.routes.ts | 108 ++++++++ backend/src/index.ts | 1 + 4 files changed, 443 insertions(+), 14 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 89ef7a7be..698d7769d 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 66; + private static currentVersion = 67; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -558,6 +558,21 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `statistics` ADD min_fee FLOAT UNSIGNED DEFAULT NULL'); await this.updateToSchemaVersion(66); } + + if (databaseSchemaVersion < 67 && config.MEMPOOL.NETWORK === "liquid") { + // Drop and re-create the elements_pegs table + await this.$executeQuery('DROP table IF EXISTS elements_pegs;'); + await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs')); + await this.$executeQuery(`UPDATE state SET number = 0 WHERE name = 'last_elements_block';`); + // Create the federation_addresses table and add the two Liquid Federation change addresses in + await this.$executeQuery(this.getCreateFederationAddressesTableQuery(), await this.$checkIfTableExists('federation_addresses')); + await this.$executeQuery(`INSERT INTO federation_addresses (bitcoinaddress) VALUES ('bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4')`); // Federation change address + await this.$executeQuery(`INSERT INTO federation_addresses (bitcoinaddress) VALUES ('3EiAcrzq1cELXScc98KeCswGWZaPGceT1d')`); // Federation change address + // Create the federation_txos table that uses the federation_addresses table as a foreign key + await this.$executeQuery(this.getCreateFederationTxosTableQuery(), await this.$checkIfTableExists('federation_txos')); + await this.$executeQuery(`INSERT INTO state VALUES('last_bitcoin_block_audit', 0, NULL);`); + await this.updateToSchemaVersion(67); + } } /** @@ -801,7 +816,31 @@ class DatabaseMigration { bitcoinaddress varchar(100) NOT NULL, bitcointxid varchar(65) NOT NULL, bitcoinindex int(11) NOT NULL, - final_tx int(11) NOT NULL + final_tx int(11) NOT NULL, + PRIMARY KEY (txid, txindex) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + + private getCreateFederationAddressesTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS federation_addresses ( + bitcoinaddress varchar(100) NOT NULL, + PRIMARY KEY (bitcoinaddress) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + + private getCreateFederationTxosTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS federation_txos ( + txid varchar(65) NOT NULL, + txindex int(11) NOT NULL, + bitcoinaddress varchar(100) NOT NULL, + amount bigint(20) unsigned NOT NULL, + blocknumber int(11) unsigned NOT NULL, + blocktime int(11) unsigned NOT NULL, + unspent tinyint(1) NOT NULL, + lastblockupdate int(11) unsigned NOT NULL, + lasttimeupdate int(11) unsigned NOT NULL, + PRIMARY KEY (txid, txindex), + FOREIGN KEY (bitcoinaddress) REFERENCES federation_addresses (bitcoinaddress) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } diff --git a/backend/src/api/liquid/elements-parser.ts b/backend/src/api/liquid/elements-parser.ts index 12439e037..842234a3a 100644 --- a/backend/src/api/liquid/elements-parser.ts +++ b/backend/src/api/liquid/elements-parser.ts @@ -5,8 +5,12 @@ import { Common } from '../common'; import DB from '../../database'; import logger from '../../logger'; +const federationChangeAddresses = ['bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4', '3EiAcrzq1cELXScc98KeCswGWZaPGceT1d']; +const auditBlockOffsetWithTip = 1; // Wait for 1 block confirmation before processing the block in the audit process to reduce the risk of reorgs + class ElementsParser { private isRunning = false; + private isUtxosUpdatingRunning = false; constructor() { } @@ -22,22 +26,19 @@ class ElementsParser { for (let height = latestBlockHeight + 1; height <= tip; height++) { const blockHash: IBitcoinApi.ChainTips = await bitcoinClient.getBlockHash(height); const block: IBitcoinApi.Block = await bitcoinClient.getBlock(blockHash, 2); + await DB.query('START TRANSACTION;'); await this.$parseBlock(block); await this.$saveLatestBlockToDatabase(block.height); + await DB.query('COMMIT;'); } this.isRunning = false; } catch (e) { + await DB.query('ROLLBACK;'); this.isRunning = false; throw new Error(e instanceof Error ? e.message : 'Error'); } } - public async $getPegDataByMonth(): Promise { - const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`; - const [rows] = await DB.query(query); - return rows; - } - protected async $parseBlock(block: IBitcoinApi.Block) { for (const tx of block.tx) { await this.$parseInputs(tx, block); @@ -55,29 +56,30 @@ class ElementsParser { protected async $parsePegIn(input: IBitcoinApi.Vin, vindex: number, txid: string, block: IBitcoinApi.Block) { const bitcoinTx: IBitcoinApi.Transaction = await bitcoinSecondClient.getRawTransaction(input.txid, true); + const bitcoinBlock: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(bitcoinTx.blockhash); const prevout = bitcoinTx.vout[input.vout || 0]; const outputAddress = prevout.scriptPubKey.address || (prevout.scriptPubKey.addresses && prevout.scriptPubKey.addresses[0]) || ''; await this.$savePegToDatabase(block.height, block.time, prevout.value * 100000000, txid, vindex, - outputAddress, bitcoinTx.txid, prevout.n, 1); + outputAddress, bitcoinTx.txid, prevout.n, bitcoinBlock.height, bitcoinBlock.time, 1); } protected async $parseOutputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) { for (const output of tx.vout) { if (output.scriptPubKey.pegout_chain) { await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n, - (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 0); + (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 0, 0, 0); } if (!output.scriptPubKey.pegout_chain && output.scriptPubKey.type === 'nulldata' && output.value && output.value > 0 && output.asset && output.asset === Common.nativeAssetId) { await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n, - (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 1); + (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 0, 0, 1); } } } protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string, - txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise { - const query = `INSERT INTO elements_pegs( + txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, bitcoinblock: number, bitcoinBlockTime: number, final_tx: number): Promise { + const query = `INSERT IGNORE INTO elements_pegs( block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`; @@ -85,7 +87,23 @@ class ElementsParser { height, blockTime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx ]; await DB.query(query, params); - logger.debug(`Saved L-BTC peg from block height #${height} with TXID ${txid}.`); + logger.debug(`Saved L-BTC peg from Liquid block height #${height} with TXID ${txid}.`); + + if (amount > 0) { // Peg-in + + // Add the address to the federation addresses table + await DB.query(`INSERT IGNORE INTO federation_addresses (bitcoinaddress) VALUES (?)`, [bitcoinaddress]); + logger.debug(`Saved new Federation address ${bitcoinaddress} to federation addresses.`); + + // Add the UTXO to the federation txos table + const query_utxos = `INSERT IGNORE INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`; + const params_utxos: (string | number)[] = [bitcointxid, bitcoinindex, bitcoinaddress, amount, bitcoinblock, bitcoinBlockTime, 1, bitcoinblock - 1, 0]; + await DB.query(query_utxos, params_utxos); + const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`) + await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']); + logger.debug(`Saved new Federation UTXO ${bitcointxid}:${bitcoinindex} belonging to ${bitcoinaddress} to federation txos.`); + + } } protected async $getLatestBlockHeightFromDatabase(): Promise { @@ -98,6 +116,269 @@ class ElementsParser { const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`; await DB.query(query, [blockHeight]); } + + ///////////// FEDERATION AUDIT ////////////// + + public async $updateFederationUtxos() { + if (this.isUtxosUpdatingRunning) { + return; + } + + this.isUtxosUpdatingRunning = true; + + try { + let auditProgress = await this.$getAuditProgress(); + // If no peg in transaction was found in the database, return + if (!auditProgress.lastBlockAudit) { + logger.debug(`No Federation UTXOs found in the database. Waiting for some to be confirmed before starting the Federation UTXOs audit.`); + this.isUtxosUpdatingRunning = false; + return; + } + + const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState(); + // If the bitcoin blockchain is not synced yet, return + if (bitcoinBlocksToSync.bitcoinHeaders > bitcoinBlocksToSync.bitcoinBlocks + 1) { + logger.debug(`Bitcoin client is not synced yet. ${bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks} blocks remaining to sync before the Federation audit process can start.`); + this.isUtxosUpdatingRunning = false; + return; + } + + auditProgress.lastBlockAudit++; + + while (auditProgress.lastBlockAudit <= auditProgress.confirmedTip) { + // First, get the current UTXOs that need to be scanned in the block + const utxos = await this.$getFederationUtxosToScan(auditProgress.lastBlockAudit); + logger.debug(`Found ${utxos.length} Federation UTXOs to scan in block ${auditProgress.lastBlockAudit} / ${auditProgress.confirmedTip}`); + + // The fast way: check if these UTXOs are still unspent as of the current block with gettxout + let spentAsTip: any[]; + let unspentAsTip: any[]; + if (auditProgress.confirmedTip - auditProgress.lastBlockAudit <= 150) { // If the audit status is not too far in the past, we can use gettxout (fast way) + const utxosToParse = await this.$getFederationUtxosToParse(utxos); + spentAsTip = utxosToParse.spentAsTip; + unspentAsTip = utxosToParse.unspentAsTip; + logger.debug(`${unspentAsTip.length} / ${utxos.length} Federation UTXOs are unspent as of tip`); + } else { // If the audit status is too far in the past, it is useless to look for still unspent txos since they will all be spent as of the tip + spentAsTip = utxos; + unspentAsTip = []; + } + + // The slow way: parse the block to look for the spending tx + logger.debug(`${spentAsTip.length} / ${utxos.length} Federation UTXOs are spent as of tip`); + + const blockHash: IBitcoinApi.ChainTips = await bitcoinSecondClient.getBlockHash(auditProgress.lastBlockAudit); + const block: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(blockHash, 2); + const nbUtxos = spentAsTip.length; + await DB.query('START TRANSACTION;'); + await this.$parseBitcoinBlock(block, spentAsTip, unspentAsTip, auditProgress.confirmedTip); + await DB.query(`COMMIT;`); + logger.debug(`Watched for spending of ${nbUtxos} Federation UTXOs in block ${auditProgress.lastBlockAudit} / ${auditProgress.confirmedTip}`); + + // Finally, update the lastblockupdate of the remaining UTXOs and save to the database + const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`) + await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']); + + auditProgress = await this.$getAuditProgress(); + auditProgress.lastBlockAudit++; + } + + this.isUtxosUpdatingRunning = false; + } catch (e) { + await DB.query('ROLLBACK;'); + this.isUtxosUpdatingRunning = false; + throw new Error(e instanceof Error ? e.message : 'Error'); + } + } + + // Get the UTXOs that need to be scanned in block height (UTXOs that were last updated in the block height - 1) + protected async $getFederationUtxosToScan(height: number) { + const query = `SELECT txid, txindex, bitcoinaddress, amount FROM federation_txos WHERE lastblockupdate = ? AND unspent = 1`; + const [rows] = await DB.query(query, [height - 1]); + return rows as any[]; + } + + // Returns the UTXOs that are spent as of tip and need to be scanned + protected async $getFederationUtxosToParse(utxos: any[]): Promise { + const spentAsTip: any[] = []; + const unspentAsTip: any[] = []; + + for (const utxo of utxos) { + const result = await bitcoinSecondClient.getTxOut(utxo.txid, utxo.txindex, false); + result ? unspentAsTip.push(utxo) : spentAsTip.push(utxo); + } + + return {spentAsTip, unspentAsTip}; + } + + protected async $parseBitcoinBlock(block: IBitcoinApi.Block, spentAsTip: any[], unspentAsTip: any[], confirmedTip: number) { + for (const tx of block.tx) { + // Check if the Federation UTXOs that was spent as of tip are spent in this block + for (const input of tx.vin) { + const txo = spentAsTip.find(txo => txo.txid === input.txid && txo.txindex === input.vout); + if (txo) { + await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, lasttimeupdate = ? WHERE txid = ? AND txindex = ?`, [block.height, block.time, txo.txid, txo.txindex]); + // Remove the TXO from the utxo array + spentAsTip.splice(spentAsTip.indexOf(txo), 1); + logger.debug(`Federation UTXO ${txo.txid}:${txo.txindex} (${txo.amount} sats) was spent in block ${block.height}.`); + } + } + // Check if an output is sent to a change address of the federation + for (const output of tx.vout) { + if (output.scriptPubKey.address && federationChangeAddresses.includes(output.scriptPubKey.address)) { + // Check that the UTXO was not already added in the DB by previous scans + const [rows_check] = await DB.query(`SELECT txid FROM federation_txos WHERE txid = ? AND txindex = ?`, [tx.txid, output.n]) as any[]; + if (rows_check.length === 0) { + const query_utxos = `INSERT INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`; + const params_utxos: (string | number)[] = [tx.txid, output.n, output.scriptPubKey.address, output.value * 100000000, block.height, block.time, 1, block.height, 0]; + await DB.query(query_utxos, params_utxos); + // Add the UTXO to the utxo array + spentAsTip.push({ + txid: tx.txid, + txindex: output.n, + bitcoinaddress: output.scriptPubKey.address, + amount: output.value * 100000000 + }); + logger.debug(`Added new Federation UTXO ${tx.txid}:${output.n} of ${output.value * 100000000} sats belonging to ${output.scriptPubKey.address} (Federation change address).`); + } + } + } + } + + for (const utxo of spentAsTip) { + await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [block.height, utxo.txid, utxo.txindex]); + } + + for (const utxo of unspentAsTip) { + await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [confirmedTip, utxo.txid, utxo.txindex]); + } + } + + protected async $saveLastBlockAuditToDatabase(blockHeight: number) { + const query = `UPDATE state SET number = ? WHERE name = 'last_bitcoin_block_audit'`; + await DB.query(query, [blockHeight]); + } + + // Get the bitcoin block where the audit process was last updated + protected async $getAuditProgress(): Promise { + const lastblockaudit = await this.$getLastBlockAudit(); + const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState(); + return { + lastBlockAudit: lastblockaudit, + confirmedTip: bitcoinBlocksToSync.bitcoinBlocks - auditBlockOffsetWithTip, + }; + } + + // Get the bitcoin blocks remaining to be synced + protected async $getBitcoinBlockchainState(): Promise { + const result = await bitcoinSecondClient.getBlockchainInfo(); + return { + bitcoinBlocks: result.blocks, + bitcoinHeaders: result.headers, + } + } + + protected async $getLastBlockAudit(): Promise { + const query = `SELECT number FROM state WHERE name = 'last_bitcoin_block_audit'`; + const [rows] = await DB.query(query); + return rows[0]['number']; + } + + ///////////// DATA QUERY ////////////// + + public async $getAuditStatus(): Promise { + const lastBlockAudit = await this.$getLastBlockAudit(); + const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState(); + return { + bitcoinBlocks: bitcoinBlocksToSync.bitcoinBlocks, + bitcoinHeaders: bitcoinBlocksToSync.bitcoinHeaders, + lastBlockAudit: lastBlockAudit, + isAuditSynced: bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks <= 2 && bitcoinBlocksToSync.bitcoinBlocks - lastBlockAudit <= 3, + }; + } + + public async $getPegDataByMonth(): Promise { + const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`; + const [rows] = await DB.query(query); + return rows; + } + + public async $getFederationReservesByMonth(): Promise { + const query = ` + SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(blocktime), '%Y-%m-01') AS date FROM federation_txos + WHERE + (blocktime > UNIX_TIMESTAMP(LAST_DAY(FROM_UNIXTIME(blocktime) - INTERVAL 1 MONTH) + INTERVAL 1 DAY)) + AND + ((unspent = 1) OR (unspent = 0 AND lasttimeupdate > UNIX_TIMESTAMP(LAST_DAY(FROM_UNIXTIME(blocktime)) + INTERVAL 1 DAY))) + GROUP BY + date;`; + const [rows] = await DB.query(query); + return rows; + } + + // Get the current L-BTC pegs and the last Liquid block it was updated + public async $getCurrentLbtcSupply(): Promise { + const [rows] = await DB.query(`SELECT SUM(amount) AS LBTC_supply FROM elements_pegs;`); + const lastblockupdate = await this.$getLatestBlockHeightFromDatabase(); + const hash = await bitcoinClient.getBlockHash(lastblockupdate); + return { + amount: rows[0]['LBTC_supply'], + lastBlockUpdate: lastblockupdate, + hash: hash + }; + } + + // Get the current reserves of the federation and the last Bitcoin block it was updated + public async $getCurrentFederationReserves(): Promise { + const [rows] = await DB.query(`SELECT SUM(amount) AS total_balance FROM federation_txos WHERE unspent = 1;`); + const lastblockaudit = await this.$getLastBlockAudit(); + const hash = await bitcoinSecondClient.getBlockHash(lastblockaudit); + return { + amount: rows[0]['total_balance'], + lastBlockUpdate: lastblockaudit, + hash: hash + }; + } + + // Get all of the federation addresses, most balances first + public async $getFederationAddresses(): Promise { + const query = `SELECT bitcoinaddress, SUM(amount) AS balance FROM federation_txos WHERE unspent = 1 GROUP BY bitcoinaddress ORDER BY balance DESC;`; + const [rows] = await DB.query(query); + return rows; + } + + // Get all of the UTXOs held by the federation, most recent first + public async $getFederationUtxos(): Promise { + const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime FROM federation_txos WHERE unspent = 1 ORDER BY blocktime DESC;`; + const [rows] = await DB.query(query); + return rows; + } + + // Get all of the federation addresses one month ago, most balances first + public async $getFederationAddressesOneMonthAgo(): Promise { + const query = ` + SELECT bitcoinaddress, SUM(amount) AS balance FROM federation_txos + WHERE + (blocktime < UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -30, CURRENT_TIMESTAMP()))) + AND + ((unspent = 1) OR (unspent = 0 AND lasttimeupdate > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -30, CURRENT_TIMESTAMP())))) + GROUP BY bitcoinaddress ORDER BY balance DESC;`; + const [rows] = await DB.query(query); + return rows; + } + + // Get all of the UTXOs held by the federation one month ago, most recent first + public async $getFederationUtxosOneMonthAgo(): Promise { + const query = ` + SELECT txid, txindex, amount, blocknumber, blocktime FROM federation_txos + WHERE + (blocktime < UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -30, CURRENT_TIMESTAMP()))) + AND + ((unspent = 1) OR (unspent = 0 AND lasttimeupdate > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -30, CURRENT_TIMESTAMP())))) + ORDER BY blocktime DESC;`; + const [rows] = await DB.query(query); + return rows; + } + } export default new ElementsParser(); diff --git a/backend/src/api/liquid/liquid.routes.ts b/backend/src/api/liquid/liquid.routes.ts index b130373e1..582b139af 100644 --- a/backend/src/api/liquid/liquid.routes.ts +++ b/backend/src/api/liquid/liquid.routes.ts @@ -15,7 +15,15 @@ class LiquidRoutes { if (config.DATABASE.ENABLED) { app + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs', this.$getElementsPegs) .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', this.$getElementsPegsByMonth) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves', this.$getFederationReserves) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/month', this.$getFederationReservesByMonth) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/addresses', this.$getFederationAddresses) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/addresses/previous-month', this.$getFederationAddressesOneMonthAgo) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos', this.$getFederationUtxos) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos/previous-month', this.$getFederationUtxosOneMonthAgo) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/status', this.$getFederationAuditStatus) ; } } @@ -63,11 +71,111 @@ class LiquidRoutes { private async $getElementsPegsByMonth(req: Request, res: Response) { try { const pegs = await elementsParser.$getPegDataByMonth(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60).toUTCString()); res.json(pegs); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } } + + private async $getFederationReservesByMonth(req: Request, res: Response) { + try { + const reserves = await elementsParser.$getFederationReservesByMonth(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60).toUTCString()); + res.json(reserves); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getElementsPegs(req: Request, res: Response) { + try { + const currentSupply = await elementsParser.$getCurrentLbtcSupply(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(currentSupply); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getFederationReserves(req: Request, res: Response) { + try { + const currentReserves = await elementsParser.$getCurrentFederationReserves(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(currentReserves); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getFederationAuditStatus(req: Request, res: Response) { + try { + const auditStatus = await elementsParser.$getAuditStatus(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(auditStatus); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getFederationAddresses(req: Request, res: Response) { + try { + const federationAddresses = await elementsParser.$getFederationAddresses(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(federationAddresses); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getFederationAddressesOneMonthAgo(req: Request, res: Response) { + try { + const federationAddresses = await elementsParser.$getFederationAddressesOneMonthAgo(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60 * 24).toUTCString()); + res.json(federationAddresses); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getFederationUtxos(req: Request, res: Response) { + try { + const federationUtxos = await elementsParser.$getFederationUtxos(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(federationUtxos); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getFederationUtxosOneMonthAgo(req: Request, res: Response) { + try { + const federationUtxos = await elementsParser.$getFederationUtxosOneMonthAgo(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60 * 24).toUTCString()); + res.json(federationUtxos); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + } export default new LiquidRoutes(); diff --git a/backend/src/index.ts b/backend/src/index.ts index a7b2ad4df..3a8449131 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -266,6 +266,7 @@ class Server { blocks.setNewBlockCallback(async () => { try { await elementsParser.$parse(); + await elementsParser.$updateFederationUtxos(); } catch (e) { logger.warn('Elements parsing error: ' + (e instanceof Error ? e.message : e)); } From de2842b62a0600de37f215847948b7f9cca240db Mon Sep 17 00:00:00 2001 From: natsee Date: Sun, 21 Jan 2024 12:46:07 +0100 Subject: [PATCH 04/40] Add pegtxid and pegindex data to federation_txos table --- backend/src/api/database-migration.ts | 2 ++ backend/src/api/liquid/elements-parser.ts | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 698d7769d..9d0a0a0d1 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -839,6 +839,8 @@ class DatabaseMigration { unspent tinyint(1) NOT NULL, lastblockupdate int(11) unsigned NOT NULL, lasttimeupdate int(11) unsigned NOT NULL, + pegtxid varchar(65) NOT NULL, + pegindex int(11) NOT NULL, PRIMARY KEY (txid, txindex), FOREIGN KEY (bitcoinaddress) REFERENCES federation_addresses (bitcoinaddress) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; diff --git a/backend/src/api/liquid/elements-parser.ts b/backend/src/api/liquid/elements-parser.ts index 842234a3a..427779898 100644 --- a/backend/src/api/liquid/elements-parser.ts +++ b/backend/src/api/liquid/elements-parser.ts @@ -96,8 +96,8 @@ class ElementsParser { logger.debug(`Saved new Federation address ${bitcoinaddress} to federation addresses.`); // Add the UTXO to the federation txos table - const query_utxos = `INSERT IGNORE INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`; - const params_utxos: (string | number)[] = [bitcointxid, bitcoinindex, bitcoinaddress, amount, bitcoinblock, bitcoinBlockTime, 1, bitcoinblock - 1, 0]; + const query_utxos = `INSERT IGNORE INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, pegtxid, pegindex) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; + const params_utxos: (string | number)[] = [bitcointxid, bitcoinindex, bitcoinaddress, amount, bitcoinblock, bitcoinBlockTime, 1, bitcoinblock - 1, 0, txid, txindex]; await DB.query(query_utxos, params_utxos); const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`) await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']); @@ -228,8 +228,8 @@ class ElementsParser { // Check that the UTXO was not already added in the DB by previous scans const [rows_check] = await DB.query(`SELECT txid FROM federation_txos WHERE txid = ? AND txindex = ?`, [tx.txid, output.n]) as any[]; if (rows_check.length === 0) { - const query_utxos = `INSERT INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`; - const params_utxos: (string | number)[] = [tx.txid, output.n, output.scriptPubKey.address, output.value * 100000000, block.height, block.time, 1, block.height, 0]; + const query_utxos = `INSERT INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, pegtxid, pegindex) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; + const params_utxos: (string | number)[] = [tx.txid, output.n, output.scriptPubKey.address, output.value * 100000000, block.height, block.time, 1, block.height, 0, '', 0]; await DB.query(query_utxos, params_utxos); // Add the UTXO to the utxo array spentAsTip.push({ @@ -348,7 +348,7 @@ class ElementsParser { // Get all of the UTXOs held by the federation, most recent first public async $getFederationUtxos(): Promise { - const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime FROM federation_txos WHERE unspent = 1 ORDER BY blocktime DESC;`; + const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex FROM federation_txos WHERE unspent = 1 ORDER BY blocktime DESC;`; const [rows] = await DB.query(query); return rows; } @@ -369,7 +369,7 @@ class ElementsParser { // Get all of the UTXOs held by the federation one month ago, most recent first public async $getFederationUtxosOneMonthAgo(): Promise { const query = ` - SELECT txid, txindex, amount, blocknumber, blocktime FROM federation_txos + SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex FROM federation_txos WHERE (blocktime < UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -30, CURRENT_TIMESTAMP()))) AND From 752eba767addb7042437d3f79a01f6b76c636e92 Mon Sep 17 00:00:00 2001 From: natsee Date: Sun, 21 Jan 2024 13:19:02 +0100 Subject: [PATCH 05/40] Liquid: add BTC reserves to L-BTC widget and make it dynamic --- .../lbtc-pegs-graph.component.ts | 43 +++-- .../app/dashboard/dashboard.component.html | 14 +- .../app/dashboard/dashboard.component.scss | 6 + .../src/app/dashboard/dashboard.component.ts | 153 ++++++++++++++---- .../src/app/interfaces/node-api.interface.ts | 29 ++++ frontend/src/app/services/api.service.ts | 34 +++- 6 files changed, 233 insertions(+), 46 deletions(-) diff --git a/frontend/src/app/components/lbtc-pegs-graph/lbtc-pegs-graph.component.ts b/frontend/src/app/components/lbtc-pegs-graph/lbtc-pegs-graph.component.ts index c4e8cbf91..f8d0843b1 100644 --- a/frontend/src/app/components/lbtc-pegs-graph/lbtc-pegs-graph.component.ts +++ b/frontend/src/app/components/lbtc-pegs-graph/lbtc-pegs-graph.component.ts @@ -41,20 +41,24 @@ export class LbtcPegsGraphComponent implements OnInit, OnChanges { } ngOnChanges() { - if (!this.data) { + if (!this.data?.liquidPegs) { return; } - this.pegsChartOptions = this.createChartOptions(this.data.series, this.data.labels); + if (!this.data.liquidReserves || this.data.liquidReserves?.series.length !== this.data.liquidPegs.series.length) { + this.pegsChartOptions = this.createChartOptions(this.data.liquidPegs.series, this.data.liquidPegs.labels); + } else { + this.pegsChartOptions = this.createChartOptions(this.data.liquidPegs.series, this.data.liquidPegs.labels, this.data.liquidReserves.series); + } } rendered() { - if (!this.data) { + if (!this.data.liquidPegs) { return; } this.isLoading = false; } - createChartOptions(series: number[], labels: string[]): EChartsOption { + createChartOptions(pegSeries: number[], labels: string[], reservesSeries?: number[],): EChartsOption { return { grid: { height: this.height, @@ -99,17 +103,18 @@ export class LbtcPegsGraphComponent implements OnInit, OnChanges { type: 'line', }, formatter: (params: any) => { - const colorSpan = (color: string) => ``; + const colorSpan = (color: string) => ``; let itemFormatted = '
' + params[0].axisValue + '
'; - params.map((item: any, index: number) => { + for (let index = params.length - 1; index >= 0; index--) { + const item = params[index]; if (index < 26) { itemFormatted += `
${colorSpan(item.color)}
-
-
${formatNumber(item.value, this.locale, '1.2-2')} L-BTC
+
+
${formatNumber(item.value, this.locale, '1.2-2')} ${item.seriesName}
`; } - }); + }; return `
${itemFormatted}
`; } }, @@ -138,20 +143,34 @@ export class LbtcPegsGraphComponent implements OnInit, OnChanges { }, series: [ { - data: series, + data: pegSeries, + name: 'L-BTC', + color: '#116761', type: 'line', stack: 'total', - smooth: false, + smooth: true, showSymbol: false, areaStyle: { opacity: 0.2, color: '#116761', }, lineStyle: { - width: 3, + width: 2, color: '#116761', }, }, + { + data: reservesSeries, + name: 'BTC', + color: '#EA983B', + type: 'line', + smooth: true, + showSymbol: false, + lineStyle: { + width: 2, + color: '#EA983B', + }, + }, ], }; } diff --git a/frontend/src/app/dashboard/dashboard.component.html b/frontend/src/app/dashboard/dashboard.component.html index 12ce14512..9603c8d93 100644 --- a/frontend/src/app/dashboard/dashboard.component.html +++ b/frontend/src/app/dashboard/dashboard.component.html @@ -33,7 +33,7 @@ - + @@ -270,8 +270,16 @@
L-BTC in circulation
- -

{{ liquidPegsMonth.series.slice(-1)[0] | number: '1.2-2' }} L-BTC

+ +

{{ (+currentPeg.amount) / 100000000 | number: '1.2-2' }} L-BTC

+ +
+
+ +
BTC Reserves
+ + +

{{ +(currentReserves.amount) / 100000000 | number: '1.2-2' }} BTC
diff --git a/frontend/src/app/dashboard/dashboard.component.scss b/frontend/src/app/dashboard/dashboard.component.scss index 884ba1027..f10c4957f 100644 --- a/frontend/src/app/dashboard/dashboard.component.scss +++ b/frontend/src/app/dashboard/dashboard.component.scss @@ -97,6 +97,12 @@ color: #ffffff66; font-size: 12px; } + .liquid-color { + color: #116761; + } + .bitcoin-color { + color: #b86d12; + } } .progress { width: 90%; diff --git a/frontend/src/app/dashboard/dashboard.component.ts b/frontend/src/app/dashboard/dashboard.component.ts index 8a34bf768..2f97b23a0 100644 --- a/frontend/src/app/dashboard/dashboard.component.ts +++ b/frontend/src/app/dashboard/dashboard.component.ts @@ -1,7 +1,7 @@ import { AfterViewInit, ChangeDetectionStrategy, Component, OnDestroy, OnInit } from '@angular/core'; -import { combineLatest, merge, Observable, of, Subscription } from 'rxjs'; -import { catchError, filter, map, scan, share, switchMap, tap } from 'rxjs/operators'; -import { BlockExtended, OptimizedMempoolStats } from '../interfaces/node-api.interface'; +import { combineLatest, concat, EMPTY, interval, merge, Observable, of, Subscription } from 'rxjs'; +import { catchError, delay, filter, map, mergeMap, scan, share, skip, startWith, switchMap, tap } from 'rxjs/operators'; +import { AuditStatus, BlockExtended, CurrentPegs, OptimizedMempoolStats } from '../interfaces/node-api.interface'; import { MempoolInfo, TransactionStripped, ReplacementInfo } from '../interfaces/websocket.interface'; import { ApiService } from '../services/api.service'; import { StateService } from '../services/state.service'; @@ -47,8 +47,15 @@ export class DashboardComponent implements OnInit, OnDestroy, AfterViewInit { transactionsWeightPerSecondOptions: any; isLoadingWebSocket$: Observable; liquidPegsMonth$: Observable; + currentPeg$: Observable; + auditStatus$: Observable; + liquidReservesMonth$: Observable; + currentReserves$: Observable; + fullHistory$: Observable; currencySubscription: Subscription; currency: string; + private lastPegBlockUpdate: number = 0; + private lastReservesBlockUpdate: number = 0; constructor( public stateService: StateService, @@ -82,35 +89,35 @@ export class DashboardComponent implements OnInit, OnDestroy, AfterViewInit { this.stateService.mempoolInfo$, this.stateService.vbytesPerSecond$ ]) - .pipe( - map(([mempoolInfo, vbytesPerSecond]) => { - const percent = Math.round((Math.min(vbytesPerSecond, this.vBytesPerSecondLimit) / this.vBytesPerSecondLimit) * 100); + .pipe( + map(([mempoolInfo, vbytesPerSecond]) => { + const percent = Math.round((Math.min(vbytesPerSecond, this.vBytesPerSecondLimit) / this.vBytesPerSecondLimit) * 100); - let progressColor = 'bg-success'; - if (vbytesPerSecond > 1667) { - progressColor = 'bg-warning'; - } - if (vbytesPerSecond > 3000) { - progressColor = 'bg-danger'; - } + let progressColor = 'bg-success'; + if (vbytesPerSecond > 1667) { + progressColor = 'bg-warning'; + } + if (vbytesPerSecond > 3000) { + progressColor = 'bg-danger'; + } - const mempoolSizePercentage = (mempoolInfo.usage / mempoolInfo.maxmempool * 100); - let mempoolSizeProgress = 'bg-danger'; - if (mempoolSizePercentage <= 50) { - mempoolSizeProgress = 'bg-success'; - } else if (mempoolSizePercentage <= 75) { - mempoolSizeProgress = 'bg-warning'; - } + const mempoolSizePercentage = (mempoolInfo.usage / mempoolInfo.maxmempool * 100); + let mempoolSizeProgress = 'bg-danger'; + if (mempoolSizePercentage <= 50) { + mempoolSizeProgress = 'bg-success'; + } else if (mempoolSizePercentage <= 75) { + mempoolSizeProgress = 'bg-warning'; + } - return { - memPoolInfo: mempoolInfo, - vBytesPerSecond: vbytesPerSecond, - progressWidth: percent + '%', - progressColor: progressColor, - mempoolSizeProgress: mempoolSizeProgress, - }; - }) - ); + return { + memPoolInfo: mempoolInfo, + vBytesPerSecond: vbytesPerSecond, + progressWidth: percent + '%', + progressColor: progressColor, + mempoolSizeProgress: mempoolSizeProgress, + }; + }) + ); this.mempoolBlocksData$ = this.stateService.mempoolBlocks$ .pipe( @@ -204,8 +211,11 @@ export class DashboardComponent implements OnInit, OnDestroy, AfterViewInit { ); if (this.stateService.network === 'liquid' || this.stateService.network === 'liquidtestnet') { - this.liquidPegsMonth$ = this.apiService.listLiquidPegsMonth$() + ////////// Pegs historical data ////////// + this.liquidPegsMonth$ = interval(60 * 60 * 1000) .pipe( + startWith(0), + switchMap(() => this.apiService.listLiquidPegsMonth$()), map((pegs) => { const labels = pegs.map(stats => stats.date); const series = pegs.map(stats => parseFloat(stats.amount) / 100000000); @@ -217,6 +227,89 @@ export class DashboardComponent implements OnInit, OnDestroy, AfterViewInit { }), share(), ); + + this.currentPeg$ = concat( + // We fetch the current peg when the page load and + // wait for the API response before listening to websocket blocks + this.apiService.liquidPegs$() + .pipe( + tap((currentPeg) => this.lastPegBlockUpdate = currentPeg.lastBlockUpdate) + ), + // Or when we receive a newer block, we wait 2 seconds so that the backend updates and we fetch the current peg + this.stateService.blocks$ + .pipe( + delay(2000), + switchMap((_) => this.apiService.liquidPegs$()), + filter((currentPeg) => currentPeg.lastBlockUpdate > this.lastPegBlockUpdate), + tap((currentPeg) => this.lastPegBlockUpdate = currentPeg.lastBlockUpdate) + ) + ).pipe( + share() + ); + + ////////// BTC Reserves historical data ////////// + this.auditStatus$ = concat( + this.apiService.federationAuditSynced$().pipe(share()), + this.stateService.blocks$.pipe( + skip(1), + delay(2000), + switchMap(() => this.apiService.federationAuditSynced$()), + share() + ) + ); + + this.liquidReservesMonth$ = interval(60 * 60 * 1000).pipe( + startWith(0), + mergeMap(() => this.apiService.federationAuditSynced$()), + switchMap((auditStatus) => { + return auditStatus.isAuditSynced ? this.apiService.listLiquidReservesMonth$() : EMPTY; + }), + map(reserves => { + const labels = reserves.map(stats => stats.date); + const series = reserves.map(stats => parseFloat(stats.amount) / 100000000); + return { + series, + labels + }; + }), + share() + ); + + this.currentReserves$ = this.auditStatus$.pipe( + filter(auditStatus => auditStatus.isAuditSynced === true), + switchMap(_ => + this.apiService.liquidReserves$().pipe( + filter((currentReserves) => currentReserves.lastBlockUpdate > this.lastReservesBlockUpdate), + tap((currentReserves) => { + this.lastReservesBlockUpdate = currentReserves.lastBlockUpdate; + }) + ) + ), + share() + ); + + this.fullHistory$ = combineLatest([this.liquidPegsMonth$, this.currentPeg$, this.liquidReservesMonth$.pipe(startWith(null)), this.currentReserves$.pipe(startWith(null))]) + .pipe( + map(([liquidPegs, currentPeg, liquidReserves, currentReserves]) => { + liquidPegs.series[liquidPegs.series.length - 1] = parseFloat(currentPeg.amount) / 100000000; + + if (liquidPegs.series.length === liquidReserves?.series.length) { + liquidReserves.series[liquidReserves.series.length - 1] = parseFloat(currentReserves?.amount) / 100000000; + } else if (liquidPegs.series.length === liquidReserves?.series.length + 1) { + liquidReserves.series.push(parseFloat(currentReserves?.amount) / 100000000); + } else { + liquidReserves = { + series: [], + labels: [] + }; + } + + return { + liquidPegs, + liquidReserves + }; + }) + ); } this.currencySubscription = this.stateService.fiatCurrency$.subscribe((fiat) => { diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 9d936722d..5989be7fa 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -76,6 +76,35 @@ export interface LiquidPegs { date: string; } +export interface CurrentPegs { + amount: string; + lastBlockUpdate: number; + hash: string; +} + +export interface FederationAddress { + address: string; + balance: string; +} + +export interface FederationUtxo { + txid: string; + txindex: number; + bitcoinaddress: string; + amount: number; + blocknumber: number; + blocktime: number; + pegtxid: string; + pegindex: number; +} + +export interface AuditStatus { + bitcoinBlocks: number; + bitcoinHeaders: number; + lastBlockAudit: number; + isAuditSynced: boolean; +} + export interface ITranslators { [language: string]: string; } /** diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 854d15c2a..42dd6a26e 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpParams, HttpResponse } from '@angular/common/http'; import { CpfpInfo, OptimizedMempoolStats, AddressInformation, LiquidPegs, ITranslators, - PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore, BlockSizesAndWeights, RbfTree, BlockAudit, Acceleration, AccelerationHistoryParams } from '../interfaces/node-api.interface'; + PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore, BlockSizesAndWeights, RbfTree, BlockAudit, Acceleration, AccelerationHistoryParams, CurrentPegs, AuditStatus, FederationAddress, FederationUtxo } from '../interfaces/node-api.interface'; import { BehaviorSubject, Observable, catchError, filter, of, shareReplay, take, tap } from 'rxjs'; import { StateService } from './state.service'; import { IBackendInfo, WebsocketResponse } from '../interfaces/websocket.interface'; @@ -178,10 +178,42 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/' + (fullRbf ? 'fullrbf/' : '') + 'replacements/' + (after || '')); } + liquidPegs$(): Observable { + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/liquid/pegs'); + } + listLiquidPegsMonth$(): Observable { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/liquid/pegs/month'); } + liquidReserves$(): Observable { + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/liquid/reserves'); + } + + listLiquidReservesMonth$(): Observable { + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/liquid/reserves/month'); + } + + federationAuditSynced$(): Observable { + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/liquid/reserves/status'); + } + + federationAddresses$(): Observable { + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/liquid/reserves/addresses'); + } + + federationUtxos$(): Observable { + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/liquid/reserves/utxos'); + } + + federationAddressesOneMonthAgo$(): Observable { + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/liquid/reserves/addresses/previous-month'); + } + + federationUtxosOneMonthAgo$(): Observable { + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/liquid/reserves/utxos/previous-month'); + } + listFeaturedAssets$(): Observable { return this.httpClient.get(this.apiBaseUrl + '/api/v1/assets/featured'); } From cd713c61b3fe403e1ac7398bbacffe1db0438d6b Mon Sep 17 00:00:00 2001 From: natsee Date: Sun, 21 Jan 2024 13:33:20 +0100 Subject: [PATCH 06/40] Liquid: wip on Federation audit dashboard --- .../components/amount/amount.component.html | 2 +- .../app/components/amount/amount.component.ts | 1 + .../liquid-master-page.component.html | 3 + .../federation-utxos-list.component.html | 84 +++++++++ .../federation-utxos-list.component.scss | 91 ++++++++++ .../federation-utxos-list.component.ts | 66 ++++++++ .../federation-utxos-stats.component.html | 46 +++++ .../federation-utxos-stats.component.scss | 80 +++++++++ .../federation-utxos-stats.component.ts | 42 +++++ .../reserves-audit-dashboard.component.html | 58 +++++++ .../reserves-audit-dashboard.component.scss | 160 ++++++++++++++++++ .../reserves-audit-dashboard.component.ts | 86 ++++++++++ .../reserves-ratio.component.html | 4 + .../reserves-ratio.component.scss | 6 + .../reserves-ratio.component.ts | 127 ++++++++++++++ .../reserves-supply-stats.component.html | 44 +++++ .../reserves-supply-stats.component.scss | 73 ++++++++ .../reserves-supply-stats.component.ts | 24 +++ frontend/src/app/graphs/echarts.ts | 4 +- .../app/liquid/liquid-master-page.module.ts | 31 ++++ 20 files changed, 1029 insertions(+), 3 deletions(-) create mode 100644 frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.html create mode 100644 frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.scss create mode 100644 frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.ts create mode 100644 frontend/src/app/components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component.html create mode 100644 frontend/src/app/components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component.scss create mode 100644 frontend/src/app/components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component.ts create mode 100644 frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.html create mode 100644 frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.scss create mode 100644 frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.ts create mode 100644 frontend/src/app/components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component.html create mode 100644 frontend/src/app/components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component.scss create mode 100644 frontend/src/app/components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component.ts create mode 100644 frontend/src/app/components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component.html create mode 100644 frontend/src/app/components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component.scss create mode 100644 frontend/src/app/components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component.ts diff --git a/frontend/src/app/components/amount/amount.component.html b/frontend/src/app/components/amount/amount.component.html index 29f61ca41..34f9be8ae 100644 --- a/frontend/src/app/components/amount/amount.component.html +++ b/frontend/src/app/components/amount/amount.component.html @@ -19,7 +19,7 @@ ‎{{ addPlus && satoshis >= 0 ? '+' : '' }}{{ satoshis / 100000000 | number : digitsInfo }} - L- + L- tL- t sBTC diff --git a/frontend/src/app/components/amount/amount.component.ts b/frontend/src/app/components/amount/amount.component.ts index 479ae4791..9c779265c 100644 --- a/frontend/src/app/components/amount/amount.component.ts +++ b/frontend/src/app/components/amount/amount.component.ts @@ -23,6 +23,7 @@ export class AmountComponent implements OnInit, OnDestroy { @Input() noFiat = false; @Input() addPlus = false; @Input() blockConversion: Price; + @Input() forceBtc: boolean = false; constructor( private stateService: StateService, diff --git a/frontend/src/app/components/liquid-master-page/liquid-master-page.component.html b/frontend/src/app/components/liquid-master-page/liquid-master-page.component.html index 49f05c3a2..30fc153c7 100644 --- a/frontend/src/app/components/liquid-master-page/liquid-master-page.component.html +++ b/frontend/src/app/components/liquid-master-page/liquid-master-page.component.html @@ -78,6 +78,9 @@ + diff --git a/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.html b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.html new file mode 100644 index 000000000..0c5981be6 --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.html @@ -0,0 +1,84 @@ +
+

Liquid Federation UTXOs

+ + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OutputAddressAmountDate
+ + + + + + + +
+ + + + + + + + + + + ‎{{ utxo.blocktime * 1000 | date:'yyyy-MM-dd HH:mm' }} +
()
+
+ + + + + + + +
+ + + + + +
+
+ +
+ +
diff --git a/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.scss b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.scss new file mode 100644 index 000000000..4208fd167 --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.scss @@ -0,0 +1,91 @@ +.spinner-border { + height: 25px; + width: 25px; + margin-top: 13px; +} + +.container-xl { + max-width: 1400px; +} +.container-xl.widget { + padding-left: 0px; + padding-bottom: 0px; +} + +tr, td, th { + border: 0px; + padding-top: 0.65rem !important; + padding-bottom: 0.6rem !important; + padding-right: 2rem !important; + .widget { + padding-right: 1rem !important; + } +} + +.clear-link { + color: white; +} + +.disabled { + pointer-events: none; + opacity: 0.5; +} + +.progress { + background-color: #2d3348; +} + +.txid { + width: 35%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + max-width: 160px; +} +.txid.widget { + width: 50%; + +} + +.address { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + max-width: 160px; + @media (max-width: 527px) { + display: none; + } +} + +.amount { + width: 12%; +} +.amount.widget { + width: 25%; +} + +.timestamp { + width: 18%; + @media (max-width: 800px) { + display: none; + } + @media (max-width: 1000px) { + .relative-time { + display: none; + } + } +} +.timestamp.widget { + width: 25%; + @media (min-width: 768px) AND (max-width: 1050px) { + display: none; + } + @media (max-width: 767px) { + display: block; + } + + @media (max-width: 500px) { + display: none; + } +} + diff --git a/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.ts b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.ts new file mode 100644 index 000000000..19122b658 --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.ts @@ -0,0 +1,66 @@ +import { Component, OnInit, ChangeDetectionStrategy, Input, ChangeDetectorRef } from '@angular/core'; +import { BehaviorSubject, combineLatest, Observable, timer, of, concat } from 'rxjs'; +import { delay, delayWhen, filter, map, retryWhen, scan, share, skip, switchMap, tap } from 'rxjs/operators'; +import { ApiService } from '../../../services/api.service'; +import { Env, StateService } from '../../../services/state.service'; +import { AuditStatus, FederationUtxo } from '../../../interfaces/node-api.interface'; +import { WebsocketService } from '../../../services/websocket.service'; + +@Component({ + selector: 'app-federation-utxos-list', + templateUrl: './federation-utxos-list.component.html', + styleUrls: ['./federation-utxos-list.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class FederationUtxosListComponent implements OnInit { + @Input() widget: boolean = false; + @Input() federationUtxos$: Observable; + + env: Env; + isLoading = true; + page = 1; + pageSize = 15; + maxSize = window.innerWidth <= 767.98 ? 3 : 5; + skeletonLines: number[] = []; + auditStatus$: Observable; + + constructor( + private apiService: ApiService, + public stateService: StateService, + private websocketService: WebsocketService, + private cd: ChangeDetectorRef, + ) { + } + + ngOnInit(): void { + this.isLoading = !this.widget; + this.env = this.stateService.env; + this.skeletonLines = this.widget === true ? [...Array(5).keys()] : [...Array(15).keys()]; + if (!this.widget) { + this.websocketService.want(['blocks']); + this.auditStatus$ = concat( + this.apiService.federationAuditSynced$(), + this.stateService.blocks$.pipe( + skip(1), + delay(2000), + switchMap(() => this.apiService.federationAuditSynced$()), + share() + ) + ); + + this.federationUtxos$ = this.auditStatus$.pipe( + filter(auditStatus => auditStatus.isAuditSynced === true), + switchMap(_ => this.apiService.federationUtxos$()), + tap(_ => this.isLoading = false), + share() + ); + } + + + } + + pageChange(page: number): void { + this.page = page; + } + +} diff --git a/frontend/src/app/components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component.html b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component.html new file mode 100644 index 000000000..84078b201 --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component.html @@ -0,0 +1,46 @@ +
+
+
+
+ +
Outputs
+ +
+
{{ federationUtxos.length }} UTXOs
+ + + +
+
+
+
Addresses
+
+
{{ federationAddresses.length }} addresses
+ + + +
+
+
+
+
+ + +
+
+
Outputs
+
+
+
+
+
+
+
Addresses
+
+
+
+
+
+
+ + diff --git a/frontend/src/app/components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component.scss b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component.scss new file mode 100644 index 000000000..50ab123b6 --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component.scss @@ -0,0 +1,80 @@ +.fee-estimation-container { + display: flex; + justify-content: space-between; + @media (min-width: 376px) { + flex-direction: row; + } + .item { + max-width: 150px; + margin: 0; + width: -webkit-fill-available; + @media (min-width: 376px) { + margin: 0 auto 0px; + } + + .card-title { + color: #4a68b9; + font-size: 10px; + margin-bottom: 4px; + font-size: 1rem; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .card-text { + font-size: 22px; + span { + font-size: 11px; + position: relative; + top: -2px; + } + } + + &:last-child { + margin-bottom: 0; + } + .card-text span { + color: #ffffff66; + font-size: 12px; + top: 0px; + } + .fee-text{ + border-bottom: 1px solid #ffffff1c; + width: fit-content; + margin: auto; + line-height: 1.45; + padding: 0px 2px; + } + .fiat { + display: block; + font-size: 14px !important; + } + } +} + +.loading-container{ + min-height: 76px; +} + +.card-text { + .skeleton-loader { + width: 100%; + display: block; + &:first-child { + max-width: 90px; + margin: 15px auto 3px; + } + &:last-child { + margin: 10px auto 3px; + max-width: 55px; + } + } +} + +.title-link, .title-link:hover, .title-link:focus, .title-link:active { + display: block; + margin-bottom: 10px; + text-decoration: none; + color: inherit; +} diff --git a/frontend/src/app/components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component.ts b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component.ts new file mode 100644 index 000000000..5f35b582e --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component.ts @@ -0,0 +1,42 @@ +import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core'; +import { concat, interval, Observable } from 'rxjs'; +import { switchMap } from 'rxjs/operators'; +import { ApiService } from '../../../services/api.service'; +import { StateService } from '../../../services/state.service'; +import { FederationAddress, FederationUtxo } from '../../../interfaces/node-api.interface'; + +@Component({ + selector: 'app-federation-utxos-stats', + templateUrl: './federation-utxos-stats.component.html', + styleUrls: ['./federation-utxos-stats.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class FederationUtxosStatsComponent implements OnInit { + @Input() federationUtxos$: Observable; + @Input() federationAddresses$: Observable; + + federationUtxosOneMonthAgo$: Observable; + federationAddressesOneMonthAgo$: Observable; + + constructor(private apiService: ApiService, private stateService: StateService) { } + + ngOnInit(): void { + + // Calls this.apiService.federationUtxosOneMonthAgo$ at load and then every day + this.federationUtxosOneMonthAgo$ = concat( + this.apiService.federationUtxosOneMonthAgo$(), + interval(24 * 60 * 60 * 1000).pipe( + switchMap(() => this.apiService.federationUtxosOneMonthAgo$()) + ) + ); + + // Calls this.apiService.federationAddressesOneMonthAgo$ at load and then every day + this.federationAddressesOneMonthAgo$ = concat( + this.apiService.federationAddressesOneMonthAgo$(), + interval(24 * 60 * 60 * 1000).pipe( + switchMap(() => this.apiService.federationAddressesOneMonthAgo$()) + ) + ); + } + +} diff --git a/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.html b/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.html new file mode 100644 index 000000000..ddac8dd1c --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.html @@ -0,0 +1,58 @@ +
+ +
+ +
+
+ BTC Reserves +
+
+
+ + +
+
+
+ +
+
+ Liquid Federation UTXOs +
+
+
+ + +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+
+
+ + + +
+ + + + +
+ Audit in progress: block {{ auditStatus.lastBlockAudit }} / {{ auditStatus.bitcoinHeaders }} +
+ + \ No newline at end of file diff --git a/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.scss b/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.scss new file mode 100644 index 000000000..46f9ffa4d --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.scss @@ -0,0 +1,160 @@ +.dashboard-container { + text-align: center; + margin-top: 0.5rem; + .col { + margin-bottom: 1.5rem; + } +} + +.card { + background-color: #1d1f31; +} + +.graph-card { + height: 100%; + @media (min-width: 992px) { + height: 385px; + } +} + +.card-title { + font-size: 1rem; + color: #4a68b9; +} +.card-title > a { + color: #4a68b9; +} + +.card-body.pool-ranking { + padding: 1.25rem 0.25rem 0.75rem 0.25rem; +} +.card-text { + font-size: 22px; +} + +#blockchain-container { + position: relative; + overflow-x: scroll; + overflow-y: hidden; + scrollbar-width: none; + -ms-overflow-style: none; +} + +#blockchain-container::-webkit-scrollbar { + display: none; +} + +.fade-border { + -webkit-mask-image: linear-gradient(to right, transparent 0%, black 10%, black 80%, transparent 100%) +} + +.main-title { + position: relative; + color: #ffffff91; + margin-top: -13px; + font-size: 11px; + text-transform: uppercase; + font-weight: 500; + text-align: center; + padding-bottom: 3px; +} + +.in-progress-message { + position: relative; + color: #ffffff91; + margin-top: 20px; + text-align: center; + padding-bottom: 3px; + font-weight: 500; +} + +.more-padding { + padding: 24px 20px !important; +} + +.card-wrapper { + .card { + height: auto !important; + } + .card-body { + display: flex; + flex: inherit; + text-align: center; + flex-direction: column; + justify-content: space-around; + padding: 22px 20px; + } +} + +.skeleton-loader { + width: 100%; + display: block; + &:first-child { + max-width: 90px; + margin: 15px auto 3px; + } + &:last-child { + margin: 10px auto 3px; + max-width: 55px; + } +} + +.card-text { + font-size: 22px; +} + +.title-link, .title-link:hover, .title-link:focus, .title-link:active { + display: block; + margin-bottom: 10px; + text-decoration: none; + color: inherit; +} + +.lastest-blocks-table { + width: 100%; + text-align: left; + tr, td, th { + border: 0px; + padding-top: 0.65rem !important; + padding-bottom: 0.8rem !important; + } + .table-cell-height { + width: 25%; + } + .table-cell-fee { + width: 25%; + text-align: right; + } + .table-cell-pool { + text-align: left; + width: 30%; + + @media (max-width: 875px) { + display: none; + } + + .pool-name { + margin-left: 1em; + } + } + .table-cell-acceleration-count { + text-align: right; + width: 20%; + } +} + +.card { + height: 385px; +} +.list-card { + height: 410px; + @media (max-width: 767px) { + height: auto; + } +} + +.mempool-block-wrapper { + max-height: 380px; + max-width: 380px; + margin: auto; +} \ No newline at end of file diff --git a/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.ts b/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.ts new file mode 100644 index 000000000..e71ff8e80 --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.ts @@ -0,0 +1,86 @@ +import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; +import { SeoService } from '../../../services/seo.service'; +import { WebsocketService } from '../../../services/websocket.service'; +import { StateService } from '../../../services/state.service'; +import { Observable, concat, delay, filter, share, skip, switchMap, tap } from 'rxjs'; +import { ApiService } from '../../../services/api.service'; +import { AuditStatus, CurrentPegs, FederationAddress, FederationUtxo } from '../../../interfaces/node-api.interface'; + +@Component({ + selector: 'app-reserves-audit-dashboard', + templateUrl: './reserves-audit-dashboard.component.html', + styleUrls: ['./reserves-audit-dashboard.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ReservesAuditDashboardComponent implements OnInit { + auditStatus$: Observable; + currentPeg$: Observable; + currentReserves$: Observable; + federationUtxos$: Observable; + federationAddresses$: Observable; + private lastPegBlockUpdate: number = 0; + private lastReservesBlockUpdate: number = 0; + + + constructor( + private seoService: SeoService, + private websocketService: WebsocketService, + private apiService: ApiService, + private stateService: StateService, + ) { + this.seoService.setTitle($localize`:@@liquid.reserves-audit:Reserves Audit Dashboard`); + } + + ngOnInit(): void { + this.websocketService.want(['blocks', 'mempool-blocks']); + + this.auditStatus$ = concat( + this.apiService.federationAuditSynced$().pipe(share()), + this.stateService.blocks$.pipe( + skip(1), + delay(2000), + switchMap(() => this.apiService.federationAuditSynced$()), + share() + ) + ); + + this.currentReserves$ = this.auditStatus$.pipe( + filter(auditStatus => auditStatus.isAuditSynced === true), + switchMap(_ => + this.apiService.liquidReserves$().pipe( + filter((currentReserves) => currentReserves.lastBlockUpdate > this.lastReservesBlockUpdate), + tap((currentReserves) => { + this.lastReservesBlockUpdate = currentReserves.lastBlockUpdate; + }) + ) + ), + share() + ); + + this.currentPeg$ = this.auditStatus$.pipe( + filter(auditStatus => auditStatus.isAuditSynced === true), + switchMap(_ => + this.apiService.liquidPegs$().pipe( + filter((currentPegs) => currentPegs.lastBlockUpdate > this.lastPegBlockUpdate), + tap((currentPegs) => { + this.lastPegBlockUpdate = currentPegs.lastBlockUpdate; + }) + ) + ), + share() + ); + + this.federationUtxos$ = this.auditStatus$.pipe( + filter(auditStatus => auditStatus.isAuditSynced === true), + switchMap(_ => this.apiService.federationUtxos$()), + share() + ); + + this.federationAddresses$ = this.auditStatus$.pipe( + filter(auditStatus => auditStatus.isAuditSynced === true), + switchMap(_ => this.apiService.federationAddresses$()), + share() + ); + } + +} diff --git a/frontend/src/app/components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component.html b/frontend/src/app/components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component.html new file mode 100644 index 000000000..561656760 --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component.html @@ -0,0 +1,4 @@ +
+
+ +
diff --git a/frontend/src/app/components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component.scss b/frontend/src/app/components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component.scss new file mode 100644 index 000000000..9881148fc --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component.scss @@ -0,0 +1,6 @@ +.loadingGraphs { + position: absolute; + top: 50%; + left: calc(50% - 16px); + z-index: 100; +} \ No newline at end of file diff --git a/frontend/src/app/components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component.ts b/frontend/src/app/components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component.ts new file mode 100644 index 000000000..0a6363257 --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component.ts @@ -0,0 +1,127 @@ +import { Component, ChangeDetectionStrategy, Input, OnChanges, OnInit } from '@angular/core'; +import { EChartsOption } from '../../../graphs/echarts'; +import { CurrentPegs } from '../../../interfaces/node-api.interface'; + + +@Component({ + selector: 'app-reserves-ratio', + templateUrl: './reserves-ratio.component.html', + styleUrls: ['./reserves-ratio.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ReservesRatioComponent implements OnInit, OnChanges { + @Input() currentPeg: CurrentPegs; + @Input() currentReserves: CurrentPegs; + pegsChartOptions: EChartsOption; + + height: number | string = '200'; + right: number | string = '10'; + top: number | string = '20'; + left: number | string = '50'; + template: ('widget' | 'advanced') = 'widget'; + isLoading = true; + + pegsChartOption: EChartsOption = {}; + pegsChartInitOption = { + renderer: 'svg' + }; + + constructor() { } + + ngOnInit() { + this.isLoading = true; + } + + ngOnChanges() { + if (!this.currentPeg || !this.currentReserves || this.currentPeg.amount === '0') { + return; + } + this.pegsChartOptions = this.createChartOptions(this.currentPeg, this.currentReserves); + } + + rendered() { + if (!this.currentPeg || !this.currentReserves) { + return; + } + this.isLoading = false; + } + + createChartOptions(currentPeg: CurrentPegs, currentReserves: CurrentPegs): EChartsOption { + return { + series: [ + { + type: 'gauge', + startAngle: 180, + endAngle: 0, + center: ['50%', '70%'], + radius: '100%', + min: 0.999, + max: 1.001, + splitNumber: 2, + axisLine: { + lineStyle: { + width: 6, + color: [ + [0.49, '#D81B60'], + [1, '#7CB342'] + ] + } + }, + axisLabel: { + color: 'inherit', + fontFamily: 'inherit', + }, + pointer: { + icon: 'path://M2090.36389,615.30999 L2090.36389,615.30999 C2091.48372,615.30999 2092.40383,616.194028 2092.44859,617.312956 L2096.90698,728.755929 C2097.05155,732.369577 2094.2393,735.416212 2090.62566,735.56078 C2090.53845,735.564269 2090.45117,735.566014 2090.36389,735.566014 L2090.36389,735.566014 C2086.74736,735.566014 2083.81557,732.63423 2083.81557,729.017692 C2083.81557,728.930412 2083.81732,728.84314 2083.82081,728.755929 L2088.2792,617.312956 C2088.32396,616.194028 2089.24407,615.30999 2090.36389,615.30999 Z', + length: '50%', + width: 16, + offsetCenter: [0, '-27%'], + itemStyle: { + color: 'auto' + } + }, + axisTick: { + length: 12, + lineStyle: { + color: 'auto', + width: 2 + } + }, + splitLine: { + length: 20, + lineStyle: { + color: 'auto', + width: 5 + } + }, + title: { + show: true, + offsetCenter: [0, '-117.5%'], + fontSize: 18, + color: '#4a68b9', + fontFamily: 'inherit', + fontWeight: 500, + }, + detail: { + fontSize: 25, + offsetCenter: [0, '-0%'], + valueAnimation: true, + fontFamily: 'inherit', + fontWeight: 500, + formatter: function (value) { + return (value).toFixed(5); + }, + color: 'inherit' + }, + data: [ + { + value: parseFloat(currentReserves.amount) / parseFloat(currentPeg.amount), + name: 'Peg-O-Meter' + } + ] + } + ] + }; + } +} + diff --git a/frontend/src/app/components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component.html b/frontend/src/app/components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component.html new file mode 100644 index 000000000..d0a624a4f --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component.html @@ -0,0 +1,44 @@ +
+
+
+
+
L-BTC in circulation
+
+
{{ (+currentPeg.amount) / 100000000 | number: '1.2-2' }} L-BTC
+ + As of block {{ currentPeg.lastBlockUpdate }} + +
+
+
+
BTC Reserves
+
+
{{ (+currentReserves.amount) / 100000000 | number: '1.2-2' }} BTC
+ + As of block {{ currentReserves.lastBlockUpdate }} + +
+
+
+
+
+ + +
+
+
L-BTC in circulation
+
+
+
+
+
+
+
BTC Reserves
+
+
+
+
+
+
+ + diff --git a/frontend/src/app/components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component.scss b/frontend/src/app/components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component.scss new file mode 100644 index 000000000..3a8a83f26 --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component.scss @@ -0,0 +1,73 @@ +.fee-estimation-container { + display: flex; + justify-content: space-between; + @media (min-width: 376px) { + flex-direction: row; + } + .item { + max-width: 150px; + margin: 0; + width: -webkit-fill-available; + @media (min-width: 376px) { + margin: 0 auto 0px; + } + + .card-title { + color: #4a68b9; + font-size: 10px; + margin-bottom: 4px; + font-size: 1rem; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .card-text { + font-size: 22px; + span { + font-size: 11px; + position: relative; + top: -2px; + } + } + + &:last-child { + margin-bottom: 0; + } + .card-text span { + color: #ffffff66; + font-size: 12px; + top: 0px; + } + .fee-text{ + border-bottom: 1px solid #ffffff1c; + width: fit-content; + margin: auto; + line-height: 1.45; + padding: 0px 2px; + } + .fiat { + display: block; + font-size: 14px !important; + } + } +} + +.loading-container{ + min-height: 76px; +} + +.card-text { + .skeleton-loader { + width: 100%; + display: block; + &:first-child { + max-width: 90px; + margin: 15px auto 3px; + } + &:last-child { + margin: 10px auto 3px; + max-width: 55px; + } + } +} \ No newline at end of file diff --git a/frontend/src/app/components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component.ts b/frontend/src/app/components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component.ts new file mode 100644 index 000000000..61f2deb8c --- /dev/null +++ b/frontend/src/app/components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component.ts @@ -0,0 +1,24 @@ +import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core'; +import { Observable } from 'rxjs'; +import { Env, StateService } from '../../../services/state.service'; +import { CurrentPegs } from '../../../interfaces/node-api.interface'; + +@Component({ + selector: 'app-reserves-supply-stats', + templateUrl: './reserves-supply-stats.component.html', + styleUrls: ['./reserves-supply-stats.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ReservesSupplyStatsComponent implements OnInit { + @Input() currentReserves$: Observable; + @Input() currentPeg$: Observable; + + env: Env; + + constructor(private stateService: StateService) { } + + ngOnInit(): void { + this.env = this.stateService.env; + } + +} diff --git a/frontend/src/app/graphs/echarts.ts b/frontend/src/app/graphs/echarts.ts index 342867168..74fec1e71 100644 --- a/frontend/src/app/graphs/echarts.ts +++ b/frontend/src/app/graphs/echarts.ts @@ -1,6 +1,6 @@ // Import tree-shakeable echarts import * as echarts from 'echarts/core'; -import { LineChart, LinesChart, BarChart, TreemapChart, PieChart, ScatterChart } from 'echarts/charts'; +import { LineChart, LinesChart, BarChart, TreemapChart, PieChart, ScatterChart, GaugeChart } from 'echarts/charts'; import { TitleComponent, TooltipComponent, GridComponent, LegendComponent, GeoComponent, DataZoomComponent, VisualMapComponent, MarkLineComponent } from 'echarts/components'; import { SVGRenderer, CanvasRenderer } from 'echarts/renderers'; // Typescript interfaces @@ -12,6 +12,6 @@ echarts.use([ TitleComponent, TooltipComponent, GridComponent, LegendComponent, GeoComponent, DataZoomComponent, VisualMapComponent, MarkLineComponent, - LineChart, LinesChart, BarChart, TreemapChart, PieChart, ScatterChart + LineChart, LinesChart, BarChart, TreemapChart, PieChart, ScatterChart, GaugeChart ]); export { echarts, EChartsOption, TreemapSeriesOption, LineSeriesOption, PieSeriesOption }; \ No newline at end of file diff --git a/frontend/src/app/liquid/liquid-master-page.module.ts b/frontend/src/app/liquid/liquid-master-page.module.ts index bb6e4cff8..90c50e0df 100644 --- a/frontend/src/app/liquid/liquid-master-page.module.ts +++ b/frontend/src/app/liquid/liquid-master-page.module.ts @@ -2,8 +2,10 @@ import { NgModule } from '@angular/core'; import { CommonModule } from '@angular/common'; import { Routes, RouterModule } from '@angular/router'; import { SharedModule } from '../shared/shared.module'; +import { NgxEchartsModule } from 'ngx-echarts'; import { LiquidMasterPageComponent } from '../components/liquid-master-page/liquid-master-page.component'; + import { StartComponent } from '../components/start/start.component'; import { AddressComponent } from '../components/address/address.component'; import { PushTransactionComponent } from '../components/push-transaction/push-transaction.component'; @@ -13,6 +15,11 @@ import { AssetsComponent } from '../components/assets/assets.component'; import { AssetsFeaturedComponent } from '../components/assets/assets-featured/assets-featured.component' import { AssetComponent } from '../components/asset/asset.component'; import { AssetsNavComponent } from '../components/assets/assets-nav/assets-nav.component'; +import { ReservesAuditDashboardComponent } from '../components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component'; +import { ReservesSupplyStatsComponent } from '../components/liquid-reserves-audit/reserves-supply-stats/reserves-supply-stats.component'; +import { FederationUtxosStatsComponent } from '../components/liquid-reserves-audit/federation-utxos-stats/federation-utxos-stats.component'; +import { FederationUtxosListComponent } from '../components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component'; +import { ReservesRatioComponent } from '../components/liquid-reserves-audit/reserves-ratio/reserves-ratio.component'; const routes: Routes = [ { @@ -64,6 +71,22 @@ const routes: Routes = [ data: { preload: true, networkSpecific: true }, loadChildren: () => import('../components/block/block.module').then(m => m.BlockModule), }, + { + path: 'audit', + data: { networks: ['liquid'] }, + component: StartComponent, + children: [ + { + path: '', + data: { networks: ['liquid'] }, + component: ReservesAuditDashboardComponent, + } + ] + }, + { + path: 'audit/utxos', + component: FederationUtxosListComponent, + }, { path: 'assets', data: { networks: ['liquid'] }, @@ -123,9 +146,17 @@ export class LiquidRoutingModule { } CommonModule, LiquidRoutingModule, SharedModule, + NgxEchartsModule.forRoot({ + echarts: () => import('../graphs/echarts').then(m => m.echarts), + }) ], declarations: [ LiquidMasterPageComponent, + ReservesAuditDashboardComponent, + ReservesSupplyStatsComponent, + FederationUtxosStatsComponent, + FederationUtxosListComponent, + ReservesRatioComponent, ] }) export class LiquidMasterPageModule { } \ No newline at end of file From 81a09e9dba772c4bcbed476f8c69a902ca99c97f Mon Sep 17 00:00:00 2001 From: natsee Date: Sun, 21 Jan 2024 13:40:31 +0100 Subject: [PATCH 07/40] Add Liquid Peg-in column to Federation UTXOs list --- .../federation-utxos-list.component.html | 6 ++++++ .../federation-utxos-list.component.scss | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.html b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.html index 0c5981be6..9edddff4c 100644 --- a/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.html +++ b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.html @@ -10,6 +10,7 @@ Output Address Amount + Liquid Peg-in Date @@ -43,6 +44,11 @@ + + + + + ‎{{ utxo.blocktime * 1000 | date:'yyyy-MM-dd HH:mm' }}
()
diff --git a/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.scss b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.scss index 4208fd167..fe458841e 100644 --- a/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.scss +++ b/frontend/src/app/components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component.scss @@ -36,7 +36,7 @@ tr, td, th { } .txid { - width: 35%; + width: 25%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; @@ -64,6 +64,14 @@ tr, td, th { width: 25%; } +.pegin { + width: 25%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + max-width: 160px; +} + .timestamp { width: 18%; @media (max-width: 800px) { From fa90eb84fc8ef304a87b4beb41b539a0ec4f3665 Mon Sep 17 00:00:00 2001 From: natsee Date: Mon, 22 Jan 2024 13:59:11 +0100 Subject: [PATCH 08/40] Truncate elements_pegs and add primary key instead of drop/create --- backend/src/api/database-migration.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 9d0a0a0d1..911576f2d 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -561,8 +561,8 @@ class DatabaseMigration { if (databaseSchemaVersion < 67 && config.MEMPOOL.NETWORK === "liquid") { // Drop and re-create the elements_pegs table - await this.$executeQuery('DROP table IF EXISTS elements_pegs;'); - await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs')); + await this.$executeQuery('TRUNCATE TABLE elements_pegs'); + await this.$executeQuery('ALTER TABLE elements_pegs ADD PRIMARY KEY (txid, txindex);'); await this.$executeQuery(`UPDATE state SET number = 0 WHERE name = 'last_elements_block';`); // Create the federation_addresses table and add the two Liquid Federation change addresses in await this.$executeQuery(this.getCreateFederationAddressesTableQuery(), await this.$checkIfTableExists('federation_addresses')); @@ -816,8 +816,7 @@ class DatabaseMigration { bitcoinaddress varchar(100) NOT NULL, bitcointxid varchar(65) NOT NULL, bitcoinindex int(11) NOT NULL, - final_tx int(11) NOT NULL, - PRIMARY KEY (txid, txindex) + final_tx int(11) NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } From 8f7cd708824df19f0464dc13a751922105149053 Mon Sep 17 00:00:00 2001 From: natsee Date: Mon, 22 Jan 2024 14:19:01 +0100 Subject: [PATCH 09/40] Add throttleTime to avoid too frequent calls to backend --- .../reserves-audit-dashboard.component.ts | 3 ++- frontend/src/app/dashboard/dashboard.component.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.ts b/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.ts index e71ff8e80..95de4fdca 100644 --- a/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.ts +++ b/frontend/src/app/components/liquid-reserves-audit/reserves-audit-dashboard/reserves-audit-dashboard.component.ts @@ -2,7 +2,7 @@ import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; import { SeoService } from '../../../services/seo.service'; import { WebsocketService } from '../../../services/websocket.service'; import { StateService } from '../../../services/state.service'; -import { Observable, concat, delay, filter, share, skip, switchMap, tap } from 'rxjs'; +import { Observable, concat, delay, filter, share, skip, switchMap, tap, throttleTime } from 'rxjs'; import { ApiService } from '../../../services/api.service'; import { AuditStatus, CurrentPegs, FederationAddress, FederationUtxo } from '../../../interfaces/node-api.interface'; @@ -38,6 +38,7 @@ export class ReservesAuditDashboardComponent implements OnInit { this.apiService.federationAuditSynced$().pipe(share()), this.stateService.blocks$.pipe( skip(1), + throttleTime(40000), delay(2000), switchMap(() => this.apiService.federationAuditSynced$()), share() diff --git a/frontend/src/app/dashboard/dashboard.component.ts b/frontend/src/app/dashboard/dashboard.component.ts index 2f97b23a0..d0f111499 100644 --- a/frontend/src/app/dashboard/dashboard.component.ts +++ b/frontend/src/app/dashboard/dashboard.component.ts @@ -1,6 +1,6 @@ import { AfterViewInit, ChangeDetectionStrategy, Component, OnDestroy, OnInit } from '@angular/core'; import { combineLatest, concat, EMPTY, interval, merge, Observable, of, Subscription } from 'rxjs'; -import { catchError, delay, filter, map, mergeMap, scan, share, skip, startWith, switchMap, tap } from 'rxjs/operators'; +import { catchError, delay, filter, map, mergeMap, scan, share, skip, startWith, switchMap, tap, throttleTime } from 'rxjs/operators'; import { AuditStatus, BlockExtended, CurrentPegs, OptimizedMempoolStats } from '../interfaces/node-api.interface'; import { MempoolInfo, TransactionStripped, ReplacementInfo } from '../interfaces/websocket.interface'; import { ApiService } from '../services/api.service'; @@ -252,6 +252,7 @@ export class DashboardComponent implements OnInit, OnDestroy, AfterViewInit { this.apiService.federationAuditSynced$().pipe(share()), this.stateService.blocks$.pipe( skip(1), + throttleTime(40000), delay(2000), switchMap(() => this.apiService.federationAuditSynced$()), share() From 392ea35d510bc2e4683c6796448ac0ae1b5a93a5 Mon Sep 17 00:00:00 2001 From: natsee Date: Mon, 22 Jan 2024 16:03:55 +0100 Subject: [PATCH 10/40] Skeleton audit dashboard --- .../liquid-master-page.component.html | 2 +- .../federation-utxos-list.component.html | 46 ++++++++++------ .../federation-utxos-list.component.scss | 8 +-- .../reserves-audit-dashboard.component.html | 52 +++++++++++++++++-- 4 files changed, 85 insertions(+), 23 deletions(-) diff --git a/frontend/src/app/components/liquid-master-page/liquid-master-page.component.html b/frontend/src/app/components/liquid-master-page/liquid-master-page.component.html index 30fc153c7..0ec9ab337 100644 --- a/frontend/src/app/components/liquid-master-page/liquid-master-page.component.html +++ b/frontend/src/app/components/liquid-master-page/liquid-master-page.component.html @@ -79,7 +79,7 @@