diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c435d6ea5..b2d34bb03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,6 +115,10 @@ jobs: - name: Sync-assets run: npm run sync-assets-dev + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MEMPOOL_CDN: 1 + VERBOSE: 1 working-directory: assets/frontend - name: Zip mining-pool assets @@ -237,6 +241,8 @@ jobs: working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MEMPOOL_CDN: 1 + VERBOSE: 1 e2e: if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" @@ -329,4 +335,32 @@ jobs: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} - \ No newline at end of file + + validate_docker_json: + if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" + runs-on: "ubuntu-latest" + name: Validate generated backend Docker JSON + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + path: docker + + - name: Install jq + run: sudo apt-get install jq -y + + - name: Create new start script to run on CI + run: | + sed '$d' start.sh > start_ci.sh + working-directory: docker/docker/backend + + - name: Run the script to generate the sample JSON + run: | + sh start_ci.sh + working-directory: docker/docker/backend + + - name: Validate JSON syntax + run: | + cat mempool-config.json | jq + working-directory: docker/docker/backend diff --git a/.gitignore b/.gitignore index 4f19f2522..381f2187c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ backend/mempool-config.json frontend/src/resources/config.template.js frontend/src/resources/config.js target +docker/backend/start_ci.sh \ No newline at end of file diff --git a/backend/.gitignore b/backend/.gitignore index b4393c2f0..5cefd4bab 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -7,6 +7,12 @@ mempool-config.json pools.json icons.json +# docker +Dockerfile +GeoIP +start.sh +wait-for-it.sh + # compiled output /dist /tmp diff --git a/backend/package-lock.json b/backend/package-lock.json index 4d16fa012..db71881cc 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -17,7 +17,7 @@ "crypto-js": "~4.2.0", "express": "~4.18.2", "maxmind": "~4.3.11", - "mysql2": "~3.7.0", + "mysql2": "~3.9.1", "redis": "^4.6.6", "rust-gbt": "file:./rust-gbt", "socks-proxy-agent": "~7.0.0", @@ -3673,9 +3673,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", + "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==", "funding": [ { "type": "individual", @@ -6110,9 +6110,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/mysql2": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.7.0.tgz", - "integrity": "sha512-c45jA3Jc1X8yJKzrWu1GpplBKGwv/wIV6ITZTlCSY7npF2YfJR+6nMP5e+NTQhUeJPSyOQAbGDCGEHbAl8HN9w==", + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.1.tgz", + "integrity": "sha512-3njoWAAhGBYy0tWBabqUQcLtczZUxrmmtc2vszQUekg3kTJyZ5/IeLC3Fo04u6y6Iy5Sba7pIIa2P/gs8D3ZeQ==", "dependencies": { "denque": "^2.1.0", "generate-function": "^2.3.1", @@ -10440,9 +10440,9 @@ "dev": true }, "follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", + "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==" }, "form-data": { "version": "4.0.0", @@ -12230,9 +12230,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "mysql2": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.7.0.tgz", - "integrity": "sha512-c45jA3Jc1X8yJKzrWu1GpplBKGwv/wIV6ITZTlCSY7npF2YfJR+6nMP5e+NTQhUeJPSyOQAbGDCGEHbAl8HN9w==", + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.1.tgz", + "integrity": "sha512-3njoWAAhGBYy0tWBabqUQcLtczZUxrmmtc2vszQUekg3kTJyZ5/IeLC3Fo04u6y6Iy5Sba7pIIa2P/gs8D3ZeQ==", "requires": { "denque": "^2.1.0", "generate-function": "^2.3.1", diff --git a/backend/package.json b/backend/package.json index cd1255392..c42f455e5 100644 --- a/backend/package.json +++ b/backend/package.json @@ -47,7 +47,7 @@ "crypto-js": "~4.2.0", "express": "~4.18.2", "maxmind": "~4.3.11", - "mysql2": "~3.7.0", + "mysql2": "~3.9.1", "rust-gbt": "file:./rust-gbt", "redis": "^4.6.6", "socks-proxy-agent": "~7.0.0", diff --git a/backend/src/__tests__/api/difficulty-adjustment.test.ts b/backend/src/__tests__/api/difficulty-adjustment.test.ts index c3e8e1a88..a365d3429 100644 --- a/backend/src/__tests__/api/difficulty-adjustment.test.ts +++ b/backend/src/__tests__/api/difficulty-adjustment.test.ts @@ -11,9 +11,35 @@ describe('Mempool Difficulty Adjustment', () => { }; const vectors = [ - [ // Vector 1 + [ // Vector 1 (normal adjustment) + [ // Inputs + dt('2024-02-02T15:42:06.000Z'), // Last DA time (in seconds) + dt('2024-02-08T14:43:05.000Z'), // timestamp of 504 blocks ago (in seconds) + dt('2024-02-11T22:43:01.000Z'), // Current time (now) (in seconds) + 830027, // Current block height + 7.333505241141637, // Previous retarget % (Passed through) + 'mainnet', // Network (if testnet, next value is non-zero) + 0, // Latest block timestamp in seconds (only used if difficulty already locked in) + ], + { // Expected Result + progressPercent: 71.97420634920636, + difficultyChange: 8.512745140778843, + estimatedRetargetDate: 1708004001715, + remainingBlocks: 565, + remainingTime: 312620715, + previousRetarget: 7.333505241141637, + previousTime: 1706888526, + nextRetargetHeight: 830592, + timeAvg: 553311, + adjustedTimeAvg: 553311, + timeOffset: 0, + expectedBlocks: 1338.0916666666667, + }, + ], + [ // Vector 2 (within quarter-epoch overlap) [ // Inputs dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) + dt('2022-08-16T03:16:54.000Z'), // timestamp of 504 blocks ago (in seconds) dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) 750134, // Current block height 0.6280047707459726, // Previous retarget % (Passed through) @@ -22,21 +48,23 @@ describe('Mempool Difficulty Adjustment', () => { ], { // Expected Result progressPercent: 9.027777777777777, - difficultyChange: 13.180707740199772, - estimatedRetargetDate: 1661895424692, + difficultyChange: 1.0420538959004633, + estimatedRetargetDate: 1662009048328, remainingBlocks: 1834, - remainingTime: 977591692, + remainingTime: 1091215328, previousRetarget: 0.6280047707459726, previousTime: 1660820820, nextRetargetHeight: 751968, timeAvg: 533038, + adjustedTimeAvg: 594992, timeOffset: 0, expectedBlocks: 161.68833333333333, }, ], - [ // Vector 2 (testnet) + [ // Vector 3 (testnet) [ // Inputs dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) + dt('2022-08-16T03:16:54.000Z'), // timestamp of 504 blocks ago (in seconds) dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) 750134, // Current block height 0.6280047707459726, // Previous retarget % (Passed through) @@ -45,22 +73,24 @@ describe('Mempool Difficulty Adjustment', () => { ], { // Expected Result is same other than timeOffset progressPercent: 9.027777777777777, - difficultyChange: 13.180707740199772, - estimatedRetargetDate: 1661895424692, + difficultyChange: 1.0420538959004633, + estimatedRetargetDate: 1662009048328, remainingBlocks: 1834, - remainingTime: 977591692, + remainingTime: 1091215328, previousTime: 1660820820, previousRetarget: 0.6280047707459726, nextRetargetHeight: 751968, timeAvg: 533038, + adjustedTimeAvg: 594992, timeOffset: -667000, // 11 min 7 seconds since last block (testnet only) // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes expectedBlocks: 161.68833333333333, }, ], - [ // Vector 3 (mainnet lock-in (epoch ending 788255)) + [ // Vector 4 (mainnet lock-in (epoch ending 788255)) [ // Inputs dt('2023-04-20T09:57:33.000Z'), // Last DA time (in seconds) + dt('2022-08-16T03:16:54.000Z'), // timestamp of 504 blocks ago (in seconds) dt('2023-05-04T14:54:09.000Z'), // Current time (now) (in seconds) 788255, // Current block height 1.7220298879531821, // Previous retarget % (Passed through) @@ -77,16 +107,17 @@ describe('Mempool Difficulty Adjustment', () => { previousTime: 1681984653, nextRetargetHeight: 788256, timeAvg: 609129, + adjustedTimeAvg: 609129, timeOffset: 0, expectedBlocks: 2045.66, }, ], - ] as [[number, number, number, number, string, number], DifficultyAdjustment][]; + ] as [[number, number, number, number, number, string, number], DifficultyAdjustment][]; for (const vector of vectors) { const result = calcDifficultyAdjustment(...vector[0]); // previousRetarget is passed through untouched - expect(result.previousRetarget).toStrictEqual(vector[0][3]); + expect(result.previousRetarget).toStrictEqual(vector[0][4]); expect(result).toStrictEqual(vector[1]); } }); diff --git a/backend/src/api/bisq/markets-api.ts b/backend/src/api/bisq/markets-api.ts index 54e0297b7..1b5b93059 100644 --- a/backend/src/api/bisq/markets-api.ts +++ b/backend/src/api/bisq/markets-api.ts @@ -646,7 +646,7 @@ class BisqMarketsApi { case 'year': return strtotime('midnight first day of january', ts); default: - throw new Error('Unsupported interval: ' + interval); + throw new Error('Unsupported interval'); } } diff --git a/backend/src/api/bitcoin/bitcoin-api.interface.ts b/backend/src/api/bitcoin/bitcoin-api.interface.ts index 3afc22897..e176566d7 100644 --- a/backend/src/api/bitcoin/bitcoin-api.interface.ts +++ b/backend/src/api/bitcoin/bitcoin-api.interface.ts @@ -106,6 +106,7 @@ export namespace IBitcoinApi { address?: string; // (string) bitcoin address addresses?: string[]; // (string) bitcoin addresses pegout_chain?: string; // (string) Elements peg-out chain + pegout_address?: string; // (string) Elements peg-out address pegout_addresses?: string[]; // (string) Elements peg-out addresses }; } diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index c0b548b9a..2f4bcee85 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -4,6 +4,7 @@ import http from 'http'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import { IEsploraApi } from './esplora-api.interface'; import logger from '../../logger'; +import { Common } from '../common'; interface FailoverHost { host: string, @@ -15,11 +16,13 @@ interface FailoverHost { outOfSync?: boolean, unreachable?: boolean, preferred?: boolean, + checked: boolean, } class FailoverRouter { activeHost: FailoverHost; fallbackHost: FailoverHost; + maxHeight: number = 0; hosts: FailoverHost[]; multihost: boolean; pollInterval: number = 60000; @@ -34,6 +37,7 @@ class FailoverRouter { this.hosts = (config.ESPLORA.FALLBACK || []).map(domain => { return { host: domain, + checked: false, rtts: [], rtt: Infinity, failures: 0, @@ -46,6 +50,7 @@ class FailoverRouter { failures: 0, socket: !!config.ESPLORA.UNIX_SOCKET_PATH, preferred: true, + checked: false, }; this.fallbackHost = this.activeHost; this.hosts.unshift(this.activeHost); @@ -74,66 +79,87 @@ class FailoverRouter { clearTimeout(this.pollTimer); } - const results = await Promise.allSettled(this.hosts.map(async (host) => { - if (host.socket) { - return this.pollConnection.get('/blocks/tip/height', { socketPath: host.host, timeout: config.ESPLORA.FALLBACK_TIMEOUT }); - } else { - return this.pollConnection.get(host.host + '/blocks/tip/height', { timeout: config.ESPLORA.FALLBACK_TIMEOUT }); - } - })); - const maxHeight = results.reduce((max, result) => Math.max(max, result.status === 'fulfilled' ? result.value?.data || 0 : 0), 0); + const start = Date.now(); // update rtts & sync status - for (let i = 0; i < results.length; i++) { - const host = this.hosts[i]; - const result = results[i].status === 'fulfilled' ? (results[i] as PromiseFulfilledResult>).value : null; - if (result) { - const height = result.data; - const rtt = result.config['meta'].rtt; - host.rtts.unshift(rtt); - host.rtts.slice(0, 5); - host.rtt = host.rtts.reduce((acc, l) => acc + l, 0) / host.rtts.length; - host.latestHeight = height; - if (height == null || isNaN(height) || (maxHeight - height > 2)) { - host.outOfSync = true; + for (const host of this.hosts) { + try { + const result = await (host.socket + ? this.pollConnection.get('/blocks/tip/height', { socketPath: host.host, timeout: config.ESPLORA.FALLBACK_TIMEOUT }) + : this.pollConnection.get(host.host + '/blocks/tip/height', { timeout: config.ESPLORA.FALLBACK_TIMEOUT }) + ); + if (result) { + const height = result.data; + this.maxHeight = Math.max(height, this.maxHeight); + const rtt = result.config['meta'].rtt; + host.rtts.unshift(rtt); + host.rtts.slice(0, 5); + host.rtt = host.rtts.reduce((acc, l) => acc + l, 0) / host.rtts.length; + host.latestHeight = height; + if (height == null || isNaN(height) || (this.maxHeight - height > 2)) { + host.outOfSync = true; + } else { + host.outOfSync = false; + } + host.unreachable = false; } else { - host.outOfSync = false; + host.outOfSync = true; + host.unreachable = true; + host.rtts = []; + host.rtt = Infinity; } - host.unreachable = false; - } else { + } catch (e) { host.outOfSync = true; host.unreachable = true; + host.rtts = []; + host.rtt = Infinity; } + host.checked = true; + + + // switch if the current host is out of sync or significantly slower than the next best alternative + const rankOrder = this.sortHosts(); + // switch if the current host is out of sync or significantly slower than the next best alternative + if (this.activeHost.outOfSync || this.activeHost.unreachable || (this.activeHost !== rankOrder[0] && rankOrder[0].preferred) || (!this.activeHost.preferred && this.activeHost.rtt > (rankOrder[0].rtt * 2) + 50)) { + if (this.activeHost.unreachable) { + logger.warn(`🚨🚨🚨 Unable to reach ${this.activeHost.host}, failing over to next best alternative 🚨🚨🚨`); + } else if (this.activeHost.outOfSync) { + logger.warn(`🚨🚨🚨 ${this.activeHost.host} has fallen behind, failing over to next best alternative 🚨🚨🚨`); + } else { + logger.debug(`🛠️ ${this.activeHost.host} is no longer the best esplora host 🛠️`); + } + this.electHost(); + } + await Common.sleep$(50); } - this.sortHosts(); + const rankOrder = this.updateFallback(); + logger.debug(`Tomahawk ranking:\n${rankOrder.map((host, index) => this.formatRanking(index, host, this.activeHost, this.maxHeight)).join('\n')}`); - logger.debug(`Tomahawk ranking:\n${this.hosts.map((host, index) => this.formatRanking(index, host, this.activeHost, maxHeight)).join('\n')}`); + const elapsed = Date.now() - start; - // switch if the current host is out of sync or significantly slower than the next best alternative - if (this.activeHost.outOfSync || this.activeHost.unreachable || (this.activeHost !== this.hosts[0] && this.hosts[0].preferred) || (!this.activeHost.preferred && this.activeHost.rtt > (this.hosts[0].rtt * 2) + 50)) { - if (this.activeHost.unreachable) { - logger.warn(`🚨🚨🚨 Unable to reach ${this.activeHost.host}, failing over to next best alternative 🚨🚨🚨`); - } else if (this.activeHost.outOfSync) { - logger.warn(`🚨🚨🚨 ${this.activeHost.host} has fallen behind, failing over to next best alternative 🚨🚨🚨`); - } else { - logger.debug(`🛠️ ${this.activeHost.host} is no longer the best esplora host 🛠️`); - } - this.electHost(); - } - - this.pollTimer = setTimeout(() => { this.pollHosts(); }, this.pollInterval); + this.pollTimer = setTimeout(() => { this.pollHosts(); }, Math.max(1, this.pollInterval - elapsed)); } private formatRanking(index: number, host: FailoverHost, active: FailoverHost, maxHeight: number): string { - const heightStatus = host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < maxHeight ? '🟧' : '✅'); - return `${host === active ? '⭐️' : ' '} ${host.rtt < Infinity ? Math.round(host.rtt).toString().padStart(5, ' ') + 'ms' : ' - '} ${host.unreachable ? '🔥' : '✅'} | block: ${host.latestHeight || '??????'} ${heightStatus} | ${host.host} ${host === active ? '⭐️' : ' '}`; + const heightStatus = !host.checked ? '⏳' : (host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < maxHeight ? '🟧' : '✅')); + return `${host === active ? '⭐️' : ' '} ${host.rtt < Infinity ? Math.round(host.rtt).toString().padStart(5, ' ') + 'ms' : ' - '} ${!host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅')} | block: ${host.latestHeight || '??????'} ${heightStatus} | ${host.host} ${host === active ? '⭐️' : ' '}`; + } + + private updateFallback(): FailoverHost[] { + const rankOrder = this.sortHosts(); + if (rankOrder.length > 1 && rankOrder[0] === this.activeHost) { + this.fallbackHost = rankOrder[1]; + } else { + this.fallbackHost = rankOrder[0]; + } + return rankOrder; } // sort hosts by connection quality, and update default fallback - private sortHosts(): void { + private sortHosts(): FailoverHost[] { // sort by connection quality - this.hosts.sort((a, b) => { + return this.hosts.slice().sort((a, b) => { if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) { if (a.preferred === b.preferred) { // lower rtt is best @@ -145,19 +171,14 @@ class FailoverRouter { return (a.unreachable || a.outOfSync) ? 1 : -1; } }); - if (this.hosts.length > 1 && this.hosts[0] === this.activeHost) { - this.fallbackHost = this.hosts[1]; - } else { - this.fallbackHost = this.hosts[0]; - } } // depose the active host and choose the next best replacement private electHost(): void { this.activeHost.outOfSync = true; this.activeHost.failures = 0; - this.sortHosts(); - this.activeHost = this.hosts[0]; + const rankOrder = this.sortHosts(); + this.activeHost = rankOrder[0]; logger.warn(`Switching esplora host to ${this.activeHost.host}`); } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 2cd043fe2..28ca38152 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,7 +2,7 @@ import config from '../config'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; @@ -37,8 +37,10 @@ class Blocks { private currentBits = 0; private lastDifficultyAdjustmentTime = 0; private previousDifficultyRetarget = 0; + private quarterEpochBlockTime: number | null = null; private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]) => Promise)[] = []; + private classifyingBlocks: boolean = false; private mainLoopTimeout: number = 120000; @@ -451,7 +453,9 @@ class Blocks { if (config.MEMPOOL.BACKEND === 'esplora') { const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx)); const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs); - await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary + if (cpfpSummary) { + await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary + } } else { await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary } @@ -565,6 +569,11 @@ class Blocks { * [INDEXING] Index transaction classification flags for Goggles */ public async $classifyBlocks(): Promise { + if (this.classifyingBlocks) { + return; + } + this.classifyingBlocks = true; + // classification requires an esplora backend if (!Common.gogglesIndexingEnabled() || config.MEMPOOL.BACKEND !== 'esplora') { return; @@ -676,6 +685,8 @@ class Blocks { indexedThisRun = 0; } } + + this.classifyingBlocks = false; } /** @@ -773,6 +784,16 @@ class Blocks { } else { this.currentBlockHeight = this.blocks[this.blocks.length - 1].height; } + if (this.currentBlockHeight >= 503) { + try { + const quarterEpochBlockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight - 503); + const quarterEpochBlock = await bitcoinApi.$getBlock(quarterEpochBlockHash); + this.quarterEpochBlockTime = quarterEpochBlock?.timestamp; + } catch (e) { + this.quarterEpochBlockTime = null; + logger.warn('failed to update last epoch block time: ' + (e instanceof Error ? e.message : e)); + } + } if (blockHeightTip - this.currentBlockHeight > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 2) { logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.MEMPOOL.INITIAL_BLOCKS_AMOUNT} recent blocks`); @@ -995,11 +1016,11 @@ class Blocks { return state; } - private updateTimerProgress(state, msg) { + private updateTimerProgress(state, msg): void { state.progress = msg; } - private clearTimer(state) { + private clearTimer(state): void { if (state.timer) { clearTimeout(state.timer); } @@ -1088,13 +1109,19 @@ class Blocks { summary = { id: hash, transactions: cpfpSummary.transactions.map(tx => { + let flags: number = 0; + try { + flags = tx.flags || Common.getTransactionFlags(tx); + } catch (e) { + logger.warn('Failed to classify transaction: ' + (e instanceof Error ? e.message : e)); + } return { txid: tx.txid, fee: tx.fee || 0, vsize: tx.vsize, value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)), rate: tx.effectiveFeePerVsize, - flags: tx.flags || Common.getTransactionFlags(tx), + flags: flags, }; }), }; @@ -1284,7 +1311,7 @@ class Blocks { return blocks; } - public async $getBlockAuditSummary(hash: string): Promise { + public async $getBlockAuditSummary(hash: string): Promise { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { return BlocksAuditsRepository.$getBlockAudit(hash); } else { @@ -1300,11 +1327,15 @@ class Blocks { return this.previousDifficultyRetarget; } + public getQuarterEpochBlockTime(): number | null { + return this.quarterEpochBlockTime; + } + public getCurrentBlockHeight(): number { return this.currentBlockHeight; } - public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise { + public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise { let transactions = txs; if (!transactions) { if (config.MEMPOOL.BACKEND === 'esplora') { @@ -1319,14 +1350,19 @@ class Blocks { } } - const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]); + if (transactions?.length != null) { + const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]); - await this.$saveCpfp(hash, height, summary); + await this.$saveCpfp(hash, height, summary); - const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); - await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); + const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); + await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); - return summary; + return summary; + } else { + logger.err(`Cannot index CPFP for block ${height} - missing transaction data`); + return null; + } } public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise { diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 208c67d70..4ca0e50d1 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -6,6 +6,7 @@ import { NodeSocket } from '../repositories/NodesSocketsRepository'; import { isIP } from 'net'; import transactionUtils from './transaction-utils'; import { isPoint } from '../utils/secp256k1'; +import logger from '../logger'; export class Common { static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ? '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49' @@ -245,7 +246,8 @@ export class Common { } else if (tx.version === 2) { flags |= TransactionFlags.v2; } - const reusedAddresses: { [address: string ]: number } = {}; + const reusedInputAddresses: { [address: string ]: number } = {}; + const reusedOutputAddresses: { [address: string ]: number } = {}; const inValues = {}; const outValues = {}; let rbf = false; @@ -261,6 +263,9 @@ export class Common { case 'v0_p2wpkh': flags |= TransactionFlags.p2wpkh; break; case 'v0_p2wsh': flags |= TransactionFlags.p2wsh; break; case 'v1_p2tr': { + if (!vin.witness?.length) { + throw new Error('Taproot input missing witness data'); + } flags |= TransactionFlags.p2tr; // in taproot, if the last witness item begins with 0x50, it's an annex const hasAnnex = vin.witness?.[vin.witness.length - 1].startsWith('50'); @@ -286,7 +291,7 @@ export class Common { } if (vin.prevout?.scriptpubkey_address) { - reusedAddresses[vin.prevout?.scriptpubkey_address] = (reusedAddresses[vin.prevout?.scriptpubkey_address] || 0) + 1; + reusedInputAddresses[vin.prevout?.scriptpubkey_address] = (reusedInputAddresses[vin.prevout?.scriptpubkey_address] || 0) + 1; } inValues[vin.prevout?.value || Math.random()] = (inValues[vin.prevout?.value || Math.random()] || 0) + 1; } @@ -301,7 +306,7 @@ export class Common { case 'p2pk': { flags |= TransactionFlags.p2pk; // detect fake pubkey (i.e. not a valid DER point on the secp256k1 curve) - hasFakePubkey = hasFakePubkey || !isPoint(vout.scriptpubkey.slice(2, -2)); + hasFakePubkey = hasFakePubkey || !isPoint(vout.scriptpubkey?.slice(2, -2)); } break; case 'multisig': { flags |= TransactionFlags.p2ms; @@ -321,7 +326,7 @@ export class Common { case 'op_return': flags |= TransactionFlags.op_return; break; } if (vout.scriptpubkey_address) { - reusedAddresses[vout.scriptpubkey_address] = (reusedAddresses[vout.scriptpubkey_address] || 0) + 1; + reusedOutputAddresses[vout.scriptpubkey_address] = (reusedOutputAddresses[vout.scriptpubkey_address] || 0) + 1; } outValues[vout.value || Math.random()] = (outValues[vout.value || Math.random()] || 0) + 1; } @@ -331,7 +336,7 @@ export class Common { // fast but bad heuristic to detect possible coinjoins // (at least 5 inputs and 5 outputs, less than half of which are unique amounts, with no address reuse) - const addressReuse = Object.values(reusedAddresses).reduce((acc, count) => Math.max(acc, count), 0) > 1; + const addressReuse = Object.keys(reusedOutputAddresses).reduce((acc, key) => Math.max(acc, (reusedInputAddresses[key] || 0) + (reusedOutputAddresses[key] || 0)), 0) > 1; if (!addressReuse && tx.vin.length >= 5 && tx.vout.length >= 5 && (Object.keys(inValues).length + Object.keys(outValues).length) <= (tx.vin.length + tx.vout.length) / 2 ) { flags |= TransactionFlags.coinjoin; } @@ -348,7 +353,12 @@ export class Common { } static classifyTransaction(tx: TransactionExtended): TransactionClassified { - const flags = Common.getTransactionFlags(tx); + let flags = 0; + try { + flags = Common.getTransactionFlags(tx); + } catch (e) { + logger.warn('Failed to add classification flags to transaction: ' + (e instanceof Error ? e.message : e)); + } tx.flags = flags; return { ...Common.stripTransaction(tx), diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 162616af6..9a5eb310a 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 67; + private static currentVersion = 68; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -566,6 +566,20 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `blocks_templates` ADD INDEX `version` (`version`)'); await this.updateToSchemaVersion(67); } + + if (databaseSchemaVersion < 68 && config.MEMPOOL.NETWORK === "liquid") { + await this.$executeQuery('TRUNCATE TABLE elements_pegs'); + await this.$executeQuery('ALTER TABLE elements_pegs ADD PRIMARY KEY (txid, txindex);'); + await this.$executeQuery(`UPDATE state SET number = 0 WHERE name = 'last_elements_block';`); + // Create the federation_addresses table and add the two Liquid Federation change addresses in + await this.$executeQuery(this.getCreateFederationAddressesTableQuery(), await this.$checkIfTableExists('federation_addresses')); + await this.$executeQuery(`INSERT INTO federation_addresses (bitcoinaddress) VALUES ('bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4')`); // Federation change address + await this.$executeQuery(`INSERT INTO federation_addresses (bitcoinaddress) VALUES ('3EiAcrzq1cELXScc98KeCswGWZaPGceT1d')`); // Federation change address + // Create the federation_txos table that uses the federation_addresses table as a foreign key + await this.$executeQuery(this.getCreateFederationTxosTableQuery(), await this.$checkIfTableExists('federation_txos')); + await this.$executeQuery(`INSERT INTO state VALUES('last_bitcoin_block_audit', 0, NULL);`); + await this.updateToSchemaVersion(68); + } } /** @@ -813,6 +827,32 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateFederationAddressesTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS federation_addresses ( + bitcoinaddress varchar(100) NOT NULL, + PRIMARY KEY (bitcoinaddress) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + + private getCreateFederationTxosTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS federation_txos ( + txid varchar(65) NOT NULL, + txindex int(11) NOT NULL, + bitcoinaddress varchar(100) NOT NULL, + amount bigint(20) unsigned NOT NULL, + blocknumber int(11) unsigned NOT NULL, + blocktime int(11) unsigned NOT NULL, + unspent tinyint(1) NOT NULL, + lastblockupdate int(11) unsigned NOT NULL, + lasttimeupdate int(11) unsigned NOT NULL, + pegtxid varchar(65) NOT NULL, + pegindex int(11) NOT NULL, + pegblocktime int(11) unsigned NOT NULL, + PRIMARY KEY (txid, txindex), + FOREIGN KEY (bitcoinaddress) REFERENCES federation_addresses (bitcoinaddress) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + private getCreatePoolsTableQuery(): string { return `CREATE TABLE IF NOT EXISTS pools ( id int(11) NOT NULL AUTO_INCREMENT, diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index d93a5f91a..2f3b21029 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -12,6 +12,7 @@ export interface DifficultyAdjustment { previousTime: number; // Unix time in ms nextRetargetHeight: number; // Block Height timeAvg: number; // Duration of time in ms + adjustedTimeAvg; // Expected block interval with hashrate implied over last 504 blocks timeOffset: number; // (Testnet) Time since last block (cap @ 20min) in ms expectedBlocks: number; // Block count } @@ -80,6 +81,7 @@ export function calcBitsDifference(oldBits: number, newBits: number): number { export function calcDifficultyAdjustment( DATime: number, + quarterEpochTime: number | null, nowSeconds: number, blockHeight: number, previousRetarget: number, @@ -100,8 +102,20 @@ export function calcDifficultyAdjustment( let difficultyChange = 0; let timeAvgSecs = blocksInEpoch ? diffSeconds / blocksInEpoch : BLOCK_SECONDS_TARGET; + let adjustedTimeAvgSecs = timeAvgSecs; + + // for the first 504 blocks of the epoch, calculate the expected avg block interval + // from a sliding window over the last 504 blocks + if (quarterEpochTime && blocksInEpoch < 503) { + const timeLastEpoch = DATime - quarterEpochTime; + const adjustedTimeLastEpoch = timeLastEpoch * (1 + (previousRetarget / 100)); + const adjustedTimeSpan = diffSeconds + adjustedTimeLastEpoch; + adjustedTimeAvgSecs = adjustedTimeSpan / 503; + difficultyChange = (BLOCK_SECONDS_TARGET / (adjustedTimeSpan / 504) - 1) * 100; + } else { + difficultyChange = (BLOCK_SECONDS_TARGET / (actualTimespan / (blocksInEpoch + 1)) - 1) * 100; + } - difficultyChange = (BLOCK_SECONDS_TARGET / (actualTimespan / (blocksInEpoch + 1)) - 1) * 100; // Max increase is x4 (+300%) if (difficultyChange > 300) { difficultyChange = 300; @@ -126,7 +140,8 @@ export function calcDifficultyAdjustment( } const timeAvg = Math.floor(timeAvgSecs * 1000); - const remainingTime = remainingBlocks * timeAvg; + const adjustedTimeAvg = Math.floor(adjustedTimeAvgSecs * 1000); + const remainingTime = remainingBlocks * adjustedTimeAvg; const estimatedRetargetDate = remainingTime + nowSeconds * 1000; return { @@ -139,6 +154,7 @@ export function calcDifficultyAdjustment( previousTime: DATime, nextRetargetHeight, timeAvg, + adjustedTimeAvg, timeOffset, expectedBlocks, }; @@ -155,9 +171,10 @@ class DifficultyAdjustmentApi { return null; } const nowSeconds = Math.floor(new Date().getTime() / 1000); + const quarterEpochBlockTime = blocks.getQuarterEpochBlockTime(); return calcDifficultyAdjustment( - DATime, nowSeconds, blockHeight, previousRetarget, + DATime, quarterEpochBlockTime, nowSeconds, blockHeight, previousRetarget, config.MEMPOOL.NETWORK, latestBlock.timestamp ); } diff --git a/backend/src/api/liquid/elements-parser.ts b/backend/src/api/liquid/elements-parser.ts index 12439e037..101a03aca 100644 --- a/backend/src/api/liquid/elements-parser.ts +++ b/backend/src/api/liquid/elements-parser.ts @@ -5,8 +5,12 @@ import { Common } from '../common'; import DB from '../../database'; import logger from '../../logger'; +const federationChangeAddresses = ['bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4', '3EiAcrzq1cELXScc98KeCswGWZaPGceT1d']; +const auditBlockOffsetWithTip = 1; // Wait for 1 block confirmation before processing the block in the audit process to reduce the risk of reorgs + class ElementsParser { private isRunning = false; + private isUtxosUpdatingRunning = false; constructor() { } @@ -32,12 +36,6 @@ class ElementsParser { } } - public async $getPegDataByMonth(): Promise { - const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`; - const [rows] = await DB.query(query); - return rows; - } - protected async $parseBlock(block: IBitcoinApi.Block) { for (const tx of block.tx) { await this.$parseInputs(tx, block); @@ -55,29 +53,30 @@ class ElementsParser { protected async $parsePegIn(input: IBitcoinApi.Vin, vindex: number, txid: string, block: IBitcoinApi.Block) { const bitcoinTx: IBitcoinApi.Transaction = await bitcoinSecondClient.getRawTransaction(input.txid, true); + const bitcoinBlock: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(bitcoinTx.blockhash); const prevout = bitcoinTx.vout[input.vout || 0]; const outputAddress = prevout.scriptPubKey.address || (prevout.scriptPubKey.addresses && prevout.scriptPubKey.addresses[0]) || ''; await this.$savePegToDatabase(block.height, block.time, prevout.value * 100000000, txid, vindex, - outputAddress, bitcoinTx.txid, prevout.n, 1); + outputAddress, bitcoinTx.txid, prevout.n, bitcoinBlock.height, bitcoinBlock.time, 1); } protected async $parseOutputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) { for (const output of tx.vout) { if (output.scriptPubKey.pegout_chain) { await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n, - (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 0); + (output.scriptPubKey.pegout_address || ''), '', 0, 0, 0, 0); } if (!output.scriptPubKey.pegout_chain && output.scriptPubKey.type === 'nulldata' && output.value && output.value > 0 && output.asset && output.asset === Common.nativeAssetId) { await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n, - (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 1); + (output.scriptPubKey.pegout_address || ''), '', 0, 0, 0, 1); } } } protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string, - txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise { - const query = `INSERT INTO elements_pegs( + txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, bitcoinblock: number, bitcoinBlockTime: number, final_tx: number): Promise { + const query = `INSERT IGNORE INTO elements_pegs( block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`; @@ -85,7 +84,22 @@ class ElementsParser { height, blockTime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx ]; await DB.query(query, params); - logger.debug(`Saved L-BTC peg from block height #${height} with TXID ${txid}.`); + logger.debug(`Saved L-BTC peg from Liquid block height #${height} with TXID ${txid}.`); + + if (amount > 0) { // Peg-in + + // Add the address to the federation addresses table + await DB.query(`INSERT IGNORE INTO federation_addresses (bitcoinaddress) VALUES (?)`, [bitcoinaddress]); + + // Add the UTXO to the federation txos table + const query_utxos = `INSERT IGNORE INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; + const params_utxos: (string | number)[] = [bitcointxid, bitcoinindex, bitcoinaddress, amount, bitcoinblock, bitcoinBlockTime, 1, bitcoinblock - 1, 0, txid, txindex, blockTime]; + await DB.query(query_utxos, params_utxos); + const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`) + await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']); + logger.debug(`Saved new Federation UTXO ${bitcointxid}:${bitcoinindex} belonging to ${bitcoinaddress} to federation txos`); + + } } protected async $getLatestBlockHeightFromDatabase(): Promise { @@ -98,6 +112,328 @@ class ElementsParser { const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`; await DB.query(query, [blockHeight]); } + + ///////////// FEDERATION AUDIT ////////////// + + public async $updateFederationUtxos() { + if (this.isUtxosUpdatingRunning) { + return; + } + + this.isUtxosUpdatingRunning = true; + + try { + let auditProgress = await this.$getAuditProgress(); + // If no peg in transaction was found in the database, return + if (!auditProgress.lastBlockAudit) { + logger.debug(`No Federation UTXOs found in the database. Waiting for some to be confirmed before starting the Federation UTXOs audit`); + this.isUtxosUpdatingRunning = false; + return; + } + + const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState(); + // If the bitcoin blockchain is not synced yet, return + if (bitcoinBlocksToSync.bitcoinHeaders > bitcoinBlocksToSync.bitcoinBlocks + 1) { + logger.debug(`Bitcoin client is not synced yet. ${bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks} blocks remaining to sync before the Federation audit process can start`); + this.isUtxosUpdatingRunning = false; + return; + } + + auditProgress.lastBlockAudit++; + + // Logging + let indexedThisRun = 0; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; + const indexingSpeeds: number[] = []; + + while (auditProgress.lastBlockAudit <= auditProgress.confirmedTip) { + + // First, get the current UTXOs that need to be scanned in the block + const utxos = await this.$getFederationUtxosToScan(auditProgress.lastBlockAudit); + + // Get the peg-out addresses that need to be scanned + const redeemAddresses = await this.$getRedeemAddressesToScan(); + + // The fast way: check if these UTXOs are still unspent as of the current block with gettxout + let spentAsTip: any[]; + let unspentAsTip: any[]; + if (auditProgress.confirmedTip - auditProgress.lastBlockAudit <= 150) { // If the audit status is not too far in the past, we can use gettxout (fast way) + const utxosToParse = await this.$getFederationUtxosToParse(utxos); + spentAsTip = utxosToParse.spentAsTip; + unspentAsTip = utxosToParse.unspentAsTip; + logger.debug(`Found ${utxos.length} Federation UTXOs and ${redeemAddresses.length} Peg-Out Addresses to scan in Bitcoin block height #${auditProgress.lastBlockAudit} / #${auditProgress.confirmedTip}`); + logger.debug(`${unspentAsTip.length} / ${utxos.length} Federation UTXOs are unspent as of tip`); + } else { // If the audit status is too far in the past, it is useless and wasteful to look for still unspent txos since they will all be spent as of the tip + spentAsTip = utxos; + unspentAsTip = []; + + // Logging + const elapsedSeconds = (Date.now() / 1000) - timer; + if (elapsedSeconds > 5) { + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; + indexingSpeeds.push(blockPerSeconds); + if (indexingSpeeds.length > 100) indexingSpeeds.shift(); // Keep the length of the up to 100 last indexing speeds + const meanIndexingSpeed = indexingSpeeds.reduce((a, b) => a + b, 0) / indexingSpeeds.length; + const eta = (auditProgress.confirmedTip - auditProgress.lastBlockAudit) / meanIndexingSpeed; + logger.debug(`Scanning ${utxos.length} Federation UTXOs and ${redeemAddresses.length} Peg-Out Addresses at Bitcoin block height #${auditProgress.lastBlockAudit} / #${auditProgress.confirmedTip} | ~${meanIndexingSpeed.toFixed(2)} blocks/sec | elapsed: ${(runningFor / 60).toFixed(0)} minutes | ETA: ${(eta / 60).toFixed(0)} minutes`); + timer = Date.now() / 1000; + indexedThisRun = 0; + } + } + + // The slow way: parse the block to look for the spending tx + const blockHash: IBitcoinApi.ChainTips = await bitcoinSecondClient.getBlockHash(auditProgress.lastBlockAudit); + const block: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(blockHash, 2); + await this.$parseBitcoinBlock(block, spentAsTip, unspentAsTip, auditProgress.confirmedTip, redeemAddresses); + + // Finally, update the lastblockupdate of the remaining UTXOs and save to the database + const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`) + await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']); + + auditProgress = await this.$getAuditProgress(); + auditProgress.lastBlockAudit++; + indexedThisRun++; + } + + this.isUtxosUpdatingRunning = false; + } catch (e) { + this.isUtxosUpdatingRunning = false; + throw new Error(e instanceof Error ? e.message : 'Error'); + } + } + + // Get the UTXOs that need to be scanned in block height (UTXOs that were last updated in the block height - 1) + protected async $getFederationUtxosToScan(height: number) { + const query = `SELECT txid, txindex, bitcoinaddress, amount FROM federation_txos WHERE lastblockupdate = ? AND unspent = 1`; + const [rows] = await DB.query(query, [height - 1]); + return rows as any[]; + } + + // Returns the UTXOs that are spent as of tip and need to be scanned + protected async $getFederationUtxosToParse(utxos: any[]): Promise { + const spentAsTip: any[] = []; + const unspentAsTip: any[] = []; + + for (const utxo of utxos) { + const result = await bitcoinSecondClient.getTxOut(utxo.txid, utxo.txindex, false); + result ? unspentAsTip.push(utxo) : spentAsTip.push(utxo); + } + + return {spentAsTip, unspentAsTip}; + } + + protected async $parseBitcoinBlock(block: IBitcoinApi.Block, spentAsTip: any[], unspentAsTip: any[], confirmedTip: number, redeemAddressesData: any[] = []) { + const redeemAddresses: string[] = redeemAddressesData.map(redeemAddress => redeemAddress.bitcoinaddress); + for (const tx of block.tx) { + let mightRedeemInThisTx = false; // If a Federation UTXO is spent in this block, we might find a peg-out address in the outputs... + // Check if the Federation UTXOs that was spent as of tip are spent in this block + for (const input of tx.vin) { + const txo = spentAsTip.find(txo => txo.txid === input.txid && txo.txindex === input.vout); + if (txo) { + mightRedeemInThisTx = true; + await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, lasttimeupdate = ? WHERE txid = ? AND txindex = ?`, [block.height, block.time, txo.txid, txo.txindex]); + // Remove the TXO from the utxo array + spentAsTip.splice(spentAsTip.indexOf(txo), 1); + logger.debug(`Federation UTXO ${txo.txid}:${txo.txindex} (${txo.amount} sats) was spent in block ${block.height}`); + } + } + // Check if an output is sent to a change address of the federation + for (const output of tx.vout) { + if (output.scriptPubKey.address && federationChangeAddresses.includes(output.scriptPubKey.address)) { + // Check that the UTXO was not already added in the DB by previous scans + const [rows_check] = await DB.query(`SELECT txid FROM federation_txos WHERE txid = ? AND txindex = ?`, [tx.txid, output.n]) as any[]; + if (rows_check.length === 0) { + const query_utxos = `INSERT INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; + const params_utxos: (string | number)[] = [tx.txid, output.n, output.scriptPubKey.address, output.value * 100000000, block.height, block.time, 1, block.height, 0, '', 0, 0]; + await DB.query(query_utxos, params_utxos); + // Add the UTXO to the utxo array + spentAsTip.push({ + txid: tx.txid, + txindex: output.n, + bitcoinaddress: output.scriptPubKey.address, + amount: output.value * 100000000 + }); + logger.debug(`Added new Federation UTXO ${tx.txid}:${output.n} (${output.value * 100000000} sats), change address: ${output.scriptPubKey.address}`); + } + } + if (mightRedeemInThisTx && output.scriptPubKey.address && redeemAddresses.includes(output.scriptPubKey.address)) { + // Find the number of times output.scriptPubKey.address appears in redeemAddresses. There can be address reuse for peg-outs... + const matchingAddress: any[] = redeemAddressesData.filter(redeemAddress => redeemAddress.bitcoinaddress === output.scriptPubKey.address && -redeemAddress.amount === Math.round(output.value * 100000000)); + if (matchingAddress.length > 0) { + if (matchingAddress.length > 1) { + // If there are more than one peg out address with the same amount, we can't know which one redeemed the UTXO: we take the oldest one + matchingAddress.sort((a, b) => a.datetime - b.datetime); + logger.debug(`Found redeem txid ${tx.txid}:${output.n} to peg-out address ${matchingAddress[0].bitcoinaddress}, amount ${matchingAddress[0].amount}, datetime ${matchingAddress[0].datetime}`); + } else { + logger.debug(`Found redeem txid ${tx.txid}:${output.n} to peg-out address ${matchingAddress[0].bitcoinaddress}, amount ${matchingAddress[0].amount}`); + } + const query_add_redeem = `UPDATE elements_pegs SET bitcointxid = ?, bitcoinindex = ? WHERE bitcoinaddress = ? AND amount = ? AND datetime = ?`; + const params_add_redeem: (string | number)[] = [tx.txid, output.n, matchingAddress[0].bitcoinaddress, matchingAddress[0].amount, matchingAddress[0].datetime]; + await DB.query(query_add_redeem, params_add_redeem); + const index = redeemAddressesData.indexOf(matchingAddress[0]); + redeemAddressesData.splice(index, 1); + redeemAddresses.splice(index, 1); + } else { // The output amount does not match the peg-out amount... log it + logger.debug(`Found redeem txid ${tx.txid}:${output.n} to peg-out address ${output.scriptPubKey.address} but output amount ${Math.round(output.value * 100000000)} does not match the peg-out amount!`); + } + } + } + } + + + for (const utxo of spentAsTip) { + await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [block.height, utxo.txid, utxo.txindex]); + } + + for (const utxo of unspentAsTip) { + await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [confirmedTip, utxo.txid, utxo.txindex]); + } + } + + protected async $saveLastBlockAuditToDatabase(blockHeight: number) { + const query = `UPDATE state SET number = ? WHERE name = 'last_bitcoin_block_audit'`; + await DB.query(query, [blockHeight]); + } + + // Get the bitcoin block where the audit process was last updated + protected async $getAuditProgress(): Promise { + const lastblockaudit = await this.$getLastBlockAudit(); + const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState(); + return { + lastBlockAudit: lastblockaudit, + confirmedTip: bitcoinBlocksToSync.bitcoinBlocks - auditBlockOffsetWithTip, + }; + } + + // Get the bitcoin blocks remaining to be synced + protected async $getBitcoinBlockchainState(): Promise { + const result = await bitcoinSecondClient.getBlockchainInfo(); + return { + bitcoinBlocks: result.blocks, + bitcoinHeaders: result.headers, + } + } + + protected async $getLastBlockAudit(): Promise { + const query = `SELECT number FROM state WHERE name = 'last_bitcoin_block_audit'`; + const [rows] = await DB.query(query); + return rows[0]['number']; + } + + protected async $getRedeemAddressesToScan(): Promise { + const query = `SELECT datetime, amount, bitcoinaddress FROM elements_pegs where amount < 0 AND bitcoinaddress != '' AND bitcointxid = '';`; + const [rows]: any[] = await DB.query(query); + return rows; + } + + ///////////// DATA QUERY ////////////// + + public async $getAuditStatus(): Promise { + const lastBlockAudit = await this.$getLastBlockAudit(); + const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState(); + return { + bitcoinBlocks: bitcoinBlocksToSync.bitcoinBlocks, + bitcoinHeaders: bitcoinBlocksToSync.bitcoinHeaders, + lastBlockAudit: lastBlockAudit, + isAuditSynced: bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks <= 2 && bitcoinBlocksToSync.bitcoinBlocks - lastBlockAudit <= 3, + }; + } + + public async $getPegDataByMonth(): Promise { + const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`; + const [rows] = await DB.query(query); + return rows; + } + + public async $getFederationReservesByMonth(): Promise { + const query = ` + SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(blocktime), '%Y-%m-01') AS date FROM federation_txos + WHERE + (blocktime > UNIX_TIMESTAMP(LAST_DAY(FROM_UNIXTIME(blocktime) - INTERVAL 1 MONTH) + INTERVAL 1 DAY)) + AND + ((unspent = 1) OR (unspent = 0 AND lasttimeupdate > UNIX_TIMESTAMP(LAST_DAY(FROM_UNIXTIME(blocktime)) + INTERVAL 1 DAY))) + GROUP BY + date;`; + const [rows] = await DB.query(query); + return rows; + } + + // Get the current L-BTC pegs and the last Liquid block it was updated + public async $getCurrentLbtcSupply(): Promise { + const [rows] = await DB.query(`SELECT SUM(amount) AS LBTC_supply FROM elements_pegs;`); + const lastblockupdate = await this.$getLatestBlockHeightFromDatabase(); + const hash = await bitcoinClient.getBlockHash(lastblockupdate); + return { + amount: rows[0]['LBTC_supply'], + lastBlockUpdate: lastblockupdate, + hash: hash + }; + } + + // Get the current reserves of the federation and the last Bitcoin block it was updated + public async $getCurrentFederationReserves(): Promise { + const [rows] = await DB.query(`SELECT SUM(amount) AS total_balance FROM federation_txos WHERE unspent = 1;`); + const lastblockaudit = await this.$getLastBlockAudit(); + const hash = await bitcoinSecondClient.getBlockHash(lastblockaudit); + return { + amount: rows[0]['total_balance'], + lastBlockUpdate: lastblockaudit, + hash: hash + }; + } + + // Get all of the federation addresses, most balances first + public async $getFederationAddresses(): Promise { + const query = `SELECT bitcoinaddress, SUM(amount) AS balance FROM federation_txos WHERE unspent = 1 GROUP BY bitcoinaddress ORDER BY balance DESC;`; + const [rows] = await DB.query(query); + return rows; + } + + // Get all of the UTXOs held by the federation, most recent first + public async $getFederationUtxos(): Promise { + const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex, pegblocktime FROM federation_txos WHERE unspent = 1 ORDER BY blocktime DESC;`; + const [rows] = await DB.query(query); + return rows; + } + + // Get the total number of federation addresses + public async $getFederationAddressesNumber(): Promise { + const query = `SELECT COUNT(DISTINCT bitcoinaddress) AS address_count FROM federation_txos WHERE unspent = 1;`; + const [rows] = await DB.query(query); + return rows[0]; + } + + // Get the total number of federation utxos + public async $getFederationUtxosNumber(): Promise { + const query = `SELECT COUNT(*) AS utxo_count FROM federation_txos WHERE unspent = 1;`; + const [rows] = await DB.query(query); + return rows[0]; + } + + // Get recent pegs in / out + public async $getPegsList(count: number = 0): Promise { + const query = `SELECT txid, txindex, amount, bitcoinaddress, bitcointxid, bitcoinindex, datetime AS blocktime FROM elements_pegs ORDER BY block DESC LIMIT 15 OFFSET ?;`; + const [rows] = await DB.query(query, [count]); + return rows; + } + + // Get all peg in / out from the last month + public async $getPegsVolumeDaily(): Promise { + const pegInQuery = await DB.query(`SELECT SUM(amount) AS volume, COUNT(*) AS number FROM elements_pegs WHERE amount > 0 and datetime > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -1, CURRENT_TIMESTAMP()));`); + const pegOutQuery = await DB.query(`SELECT SUM(amount) AS volume, COUNT(*) AS number FROM elements_pegs WHERE amount < 0 and datetime > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -1, CURRENT_TIMESTAMP()));`); + return [ + pegInQuery[0][0], + pegOutQuery[0][0] + ]; + } + + // Get the total pegs number + public async $getPegsCount(): Promise { + const [rows] = await DB.query(`SELECT COUNT(*) AS pegs_count FROM elements_pegs;`); + return rows[0]; + } } export default new ElementsParser(); diff --git a/backend/src/api/liquid/liquid.routes.ts b/backend/src/api/liquid/liquid.routes.ts index b130373e1..01d91e3b0 100644 --- a/backend/src/api/liquid/liquid.routes.ts +++ b/backend/src/api/liquid/liquid.routes.ts @@ -15,7 +15,18 @@ class LiquidRoutes { if (config.DATABASE.ENABLED) { app + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs', this.$getElementsPegs) .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', this.$getElementsPegsByMonth) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/list/:count', this.$getPegsList) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/volume', this.$getPegsVolumeDaily) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/count', this.$getPegsCount) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves', this.$getFederationReserves) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/month', this.$getFederationReservesByMonth) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/addresses', this.$getFederationAddresses) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/addresses/total', this.$getFederationAddressesNumber) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos', this.$getFederationUtxos) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos/total', this.$getFederationUtxosNumber) + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/status', this.$getFederationAuditStatus) ; } } @@ -63,11 +74,147 @@ class LiquidRoutes { private async $getElementsPegsByMonth(req: Request, res: Response) { try { const pegs = await elementsParser.$getPegDataByMonth(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60).toUTCString()); res.json(pegs); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } } + + private async $getFederationReservesByMonth(req: Request, res: Response) { + try { + const reserves = await elementsParser.$getFederationReservesByMonth(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60).toUTCString()); + res.json(reserves); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getElementsPegs(req: Request, res: Response) { + try { + const currentSupply = await elementsParser.$getCurrentLbtcSupply(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(currentSupply); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getFederationReserves(req: Request, res: Response) { + try { + const currentReserves = await elementsParser.$getCurrentFederationReserves(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(currentReserves); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getFederationAuditStatus(req: Request, res: Response) { + try { + const auditStatus = await elementsParser.$getAuditStatus(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(auditStatus); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getFederationAddresses(req: Request, res: Response) { + try { + const federationAddresses = await elementsParser.$getFederationAddresses(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(federationAddresses); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getFederationAddressesNumber(req: Request, res: Response) { + try { + const federationAddresses = await elementsParser.$getFederationAddressesNumber(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(federationAddresses); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getFederationUtxos(req: Request, res: Response) { + try { + const federationUtxos = await elementsParser.$getFederationUtxos(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(federationUtxos); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getFederationUtxosNumber(req: Request, res: Response) { + try { + const federationUtxos = await elementsParser.$getFederationUtxosNumber(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(federationUtxos); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getPegsList(req: Request, res: Response) { + try { + const recentPegs = await elementsParser.$getPegsList(parseInt(req.params?.count)); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(recentPegs); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getPegsVolumeDaily(req: Request, res: Response) { + try { + const pegsVolume = await elementsParser.$getPegsVolumeDaily(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(pegsVolume); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getPegsCount(req: Request, res: Response) { + try { + const pegsCount = await elementsParser.$getPegsCount(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + res.json(pegsCount); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + } export default new LiquidRoutes(); diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 58921fcfb..b9da7d4e8 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,6 +1,6 @@ import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt'; import logger from '../logger'; -import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag, TransactionClassified } from '../mempool.interfaces'; +import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag, TransactionClassified, TransactionCompressed, MempoolDeltaChange } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; @@ -171,7 +171,7 @@ class MempoolBlocks { for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) { let added: TransactionClassified[] = []; let removed: string[] = []; - const changed: { txid: string, rate: number | undefined, acc: boolean | undefined }[] = []; + const changed: TransactionClassified[] = []; if (mempoolBlocks[i] && !prevBlocks[i]) { added = mempoolBlocks[i].transactions; } else if (!mempoolBlocks[i] && prevBlocks[i]) { @@ -194,14 +194,14 @@ class MempoolBlocks { if (!prevIds[tx.txid]) { added.push(tx); } else if (tx.rate !== prevIds[tx.txid].rate || tx.acc !== prevIds[tx.txid].acc) { - changed.push({ txid: tx.txid, rate: tx.rate, acc: tx.acc }); + changed.push(tx); } }); } mempoolBlockDeltas.push({ - added, + added: added.map(this.compressTx), removed, - changed, + changed: changed.map(this.compressDeltaChange), }); } return mempoolBlockDeltas; @@ -691,6 +691,38 @@ class MempoolBlocks { }); return { blocks: convertedBlocks, blockWeights, rates: convertedRates, clusters: convertedClusters, overflow: convertedOverflow }; } + + public compressTx(tx: TransactionClassified): TransactionCompressed { + if (tx.acc) { + return [ + tx.txid, + tx.fee, + tx.vsize, + tx.value, + Math.round((tx.rate || (tx.fee / tx.vsize)) * 100) / 100, + tx.flags, + 1 + ]; + } else { + return [ + tx.txid, + tx.fee, + tx.vsize, + tx.value, + Math.round((tx.rate || (tx.fee / tx.vsize)) * 100) / 100, + tx.flags, + ]; + } + } + + public compressDeltaChange(tx: TransactionClassified): MempoolDeltaChange { + return [ + tx.txid, + Math.round((tx.rate || (tx.fee / tx.vsize)) * 100) / 100, + tx.flags, + tx.acc ? 1 : 0, + ]; + } } export default new MempoolBlocks(); diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index b23ad04c5..85554db2d 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -142,7 +142,7 @@ class Mining { public async $getPoolStat(slug: string): Promise { const pool = await PoolsRepository.$getPool(slug); if (!pool) { - throw new Error('This mining pool does not exist ' + escape(slug)); + throw new Error('This mining pool does not exist'); } const blockCount: number = await BlocksRepository.$blockCount(pool.id); diff --git a/backend/src/api/statistics/statistics-api.ts b/backend/src/api/statistics/statistics-api.ts index 5c6896619..c7c3f37b0 100644 --- a/backend/src/api/statistics/statistics-api.ts +++ b/backend/src/api/statistics/statistics-api.ts @@ -285,7 +285,7 @@ class StatisticsApi { public async $list2H(): Promise { try { - const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 120`; + const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL 2 HOUR) AND NOW() ORDER BY statistics.added DESC`; const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout }); return this.mapStatisticToOptimizedStatistic(rows as Statistic[]); } catch (e) { @@ -296,7 +296,7 @@ class StatisticsApi { public async $list24H(): Promise { try { - const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 1440`; + const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL 24 HOUR) AND NOW() ORDER BY statistics.added DESC`; const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout }); return this.mapStatisticToOptimizedStatistic(rows as Statistic[]); } catch (e) { diff --git a/backend/src/api/statistics/statistics.ts b/backend/src/api/statistics/statistics.ts index 494777aad..2926a4b17 100644 --- a/backend/src/api/statistics/statistics.ts +++ b/backend/src/api/statistics/statistics.ts @@ -6,6 +6,7 @@ import statisticsApi from './statistics-api'; class Statistics { protected intervalTimer: NodeJS.Timer | undefined; + protected lastRun: number = 0; protected newStatisticsEntryCallback: ((stats: OptimizedStatistic) => void) | undefined; public setNewStatisticsEntryCallback(fn: (stats: OptimizedStatistic) => void) { @@ -23,15 +24,21 @@ class Statistics { setTimeout(() => { this.runStatistics(); this.intervalTimer = setInterval(() => { - this.runStatistics(); + this.runStatistics(true); }, 1 * 60 * 1000); }, difference); } - private async runStatistics(): Promise { + public async runStatistics(skipIfRecent = false): Promise { if (!memPool.isInSync()) { return; } + + if (skipIfRecent && new Date().getTime() / 1000 - this.lastRun < 30) { + return; + } + + this.lastRun = new Date().getTime() / 1000; const currentMempool = memPool.getMempool(); const txPerSecond = memPool.getTxPerSecond(); const vBytesPerSecond = memPool.getVBytesPerSecond(); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index d0e0b7fd8..b78389b64 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -23,6 +23,7 @@ import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; import accelerationApi from './services/acceleration'; import mempool from './mempool'; +import statistics from './statistics/statistics'; interface AddressTransactions { mempool: MempoolTransactionExtended[], @@ -259,7 +260,7 @@ class WebsocketHandler { const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions(); response['projected-block-transactions'] = JSON.stringify({ index: index, - blockTransactions: mBlocksWithTransactions[index]?.transactions || [], + blockTransactions: (mBlocksWithTransactions[index]?.transactions || []).map(mempoolBlocks.compressTx), }); } else { client['track-mempool-block'] = null; @@ -723,6 +724,7 @@ class WebsocketHandler { } this.printLogs(); + await statistics.runStatistics(); const _memPool = memPool.getMempool(); @@ -999,7 +1001,7 @@ class WebsocketHandler { if (mBlockDeltas[index].added.length > (mBlocksWithTransactions[index]?.transactions.length / 2)) { response['projected-block-transactions'] = getCachedResponse(`projected-block-transactions-full-${index}`, { index: index, - blockTransactions: mBlocksWithTransactions[index].transactions, + blockTransactions: mBlocksWithTransactions[index].transactions.map(mempoolBlocks.compressTx), }); } else { response['projected-block-transactions'] = getCachedResponse(`projected-block-transactions-delta-${index}`, { @@ -1014,6 +1016,8 @@ class WebsocketHandler { client.send(this.serializeResponse(response)); } }); + + await statistics.runStatistics(); } // takes a dictionary of JSON serialized values diff --git a/backend/src/index.ts b/backend/src/index.ts index a7b2ad4df..3a8449131 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -266,6 +266,7 @@ class Server { blocks.setNewBlockCallback(async () => { try { await elementsParser.$parse(); + await elementsParser.$updateFederationUtxos(); } catch (e) { logger.warn('Elements parsing error: ' + (e instanceof Error ? e.message : e)); } diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 90b4a59e6..dcb91d010 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -185,7 +185,8 @@ class Indexer { await blocks.$generateCPFPDatabase(); await blocks.$generateAuditStats(); await auditReplicator.$sync(); - await blocks.$classifyBlocks(); + // do not wait for classify blocks to finish + blocks.$classifyBlocks(); } catch (e) { this.indexerRunning = false; logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index ead0a84ad..71612f25f 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -65,9 +65,9 @@ export interface MempoolBlockWithTransactions extends MempoolBlock { } export interface MempoolBlockDelta { - added: TransactionClassified[]; + added: TransactionCompressed[]; removed: string[]; - changed: { txid: string, rate: number | undefined, flags?: number }[]; + changed: MempoolDeltaChange[]; } interface VinStrippedToScriptsig { @@ -196,6 +196,11 @@ export interface TransactionClassified extends TransactionStripped { flags: number; } +// [txid, fee, vsize, value, rate, flags, acceleration?] +export type TransactionCompressed = [string, number, number, number, number, number, 1?]; +// [txid, rate, flags, acceleration?] +export type MempoolDeltaChange = [string, number, number, (1|0)]; + // binary flags for transaction classification export const TransactionFlags = { // features diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index c17958d2b..62f28c56f 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -59,7 +59,7 @@ class BlocksAuditRepositories { } } - public async $getBlockAudit(hash: string): Promise { + public async $getBlockAudit(hash: string): Promise { try { const [rows]: any[] = await DB.query( `SELECT blocks_audits.height, blocks_audits.hash as id, UNIX_TIMESTAMP(blocks_audits.time) as timestamp, @@ -75,8 +75,8 @@ class BlocksAuditRepositories { expected_weight as expectedWeight FROM blocks_audits JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash - WHERE blocks_audits.hash = "${hash}" - `); + WHERE blocks_audits.hash = ? + `, [hash]); if (rows.length) { rows[0].missingTxs = JSON.parse(rows[0].missingTxs); @@ -101,8 +101,8 @@ class BlocksAuditRepositories { const [rows]: any[] = await DB.query( `SELECT hash, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight FROM blocks_audits - WHERE blocks_audits.hash = "${hash}" - `); + WHERE blocks_audits.hash = ? + `, [hash]); return rows[0]; } catch (e: any) { logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index a2a084265..e6e92d60f 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -5,7 +5,7 @@ import logger from '../logger'; import { Common } from '../api/common'; import PoolsRepository from './PoolsRepository'; import HashratesRepository from './HashratesRepository'; -import { escape } from 'mysql2'; +import { RowDataPacket, escape } from 'mysql2'; import BlocksSummariesRepository from './BlocksSummariesRepository'; import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository'; import bitcoinClient from '../api/bitcoin/bitcoin-client'; @@ -478,7 +478,7 @@ class BlocksRepository { public async $getBlocksByPool(slug: string, startHeight?: number): Promise { const pool = await PoolsRepository.$getPool(slug); if (!pool) { - throw new Error('This mining pool does not exist ' + escape(slug)); + throw new Error('This mining pool does not exist'); } const params: any[] = []; @@ -802,10 +802,10 @@ class BlocksRepository { /** * Get a list of blocks that have been indexed */ - public async $getIndexedBlocks(): Promise { + public async $getIndexedBlocks(): Promise<{ height: number, hash: string }[]> { try { - const [rows]: any = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`); - return rows; + const [rows] = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`) as RowDataPacket[][]; + return rows as { height: number, hash: string }[]; } catch (e) { logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e)); throw e; @@ -815,7 +815,7 @@ class BlocksRepository { /** * Get a list of blocks that have not had CPFP data indexed */ - public async $getCPFPUnindexedBlocks(): Promise { + public async $getCPFPUnindexedBlocks(): Promise { try { const blockchainInfo = await bitcoinClient.getBlockchainInfo(); const currentBlockHeight = blockchainInfo.blocks; @@ -825,13 +825,13 @@ class BlocksRepository { } const minHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1); - const [rows]: any[] = await DB.query(` + const [rows] = await DB.query(` SELECT height FROM compact_cpfp_clusters WHERE height <= ? AND height >= ? GROUP BY height ORDER BY height DESC; - `, [currentBlockHeight, minHeight]); + `, [currentBlockHeight, minHeight]) as RowDataPacket[][]; const indexedHeights = {}; rows.forEach((row) => { indexedHeights[row.height] = true; }); diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index f85914e31..63ad5ddf2 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -1,3 +1,4 @@ +import { RowDataPacket } from 'mysql2'; import DB from '../database'; import logger from '../logger'; import { BlockSummary, TransactionClassified } from '../mempool.interfaces'; @@ -69,7 +70,7 @@ class BlocksSummariesRepository { public async $getIndexedSummariesId(): Promise { try { - const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`); + const [rows] = await DB.query(`SELECT id from blocks_summaries`) as RowDataPacket[][]; return rows.map(row => row.id); } catch (e) { logger.err(`Cannot get block summaries id list. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 96cbf6f75..ec44afebe 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -139,7 +139,7 @@ class HashratesRepository { public async $getPoolWeeklyHashrate(slug: string): Promise { const pool = await PoolsRepository.$getPool(slug); if (!pool) { - throw new Error('This mining pool does not exist ' + escape(slug)); + throw new Error('This mining pool does not exist'); } // Find hashrate boundaries diff --git a/backend/src/utils/secp256k1.ts b/backend/src/utils/secp256k1.ts index cc731f17d..9e0f6dc3b 100644 --- a/backend/src/utils/secp256k1.ts +++ b/backend/src/utils/secp256k1.ts @@ -31,6 +31,9 @@ const curveP = BigInt(`0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF * @returns {boolean} true if the point is on the SECP256K1 curve */ export function isPoint(pointHex: string): boolean { + if (!pointHex?.length) { + return false; + } if ( !( // is uncompressed diff --git a/contributors/jamesblacklock.txt b/contributors/jamesblacklock.txt new file mode 100644 index 000000000..11591f451 --- /dev/null +++ b/contributors/jamesblacklock.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of December 20, 2023. + +Signed: jamesblacklock diff --git a/contributors/natsee.txt b/contributors/natsoni.txt similarity index 90% rename from contributors/natsee.txt rename to contributors/natsoni.txt index c391ce823..ac1007ecf 100644 --- a/contributors/natsee.txt +++ b/contributors/natsoni.txt @@ -1,3 +1,3 @@ I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of November 16, 2023. -Signed: natsee +Signed: natsoni diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index aefb095cf..8f69fd0c1 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -35,7 +35,7 @@ "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", - "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__ + "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__, "MAX_TRACKED_ADDRESSES": __MEMPOOL_MAX_TRACKED_ADDRESSES__ }, "CORE_RPC": { diff --git a/docker/backend/start.sh b/docker/backend/start.sh index ce8f72368..ba9b99233 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -55,7 +55,7 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false} # ESPLORA __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000} -__ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:="null"} +__ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:=""} __ESPLORA_BATCH_QUERY_BASE_SIZE__=${ESPLORA_BATCH_QUERY_BASE_SIZE:=1000} __ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000} __ESPLORA_REQUEST_TIMEOUT__=${ESPLORA_REQUEST_TIMEOUT:=5000} diff --git a/frontend/.gitignore b/frontend/.gitignore index 8159e7c7b..d2a765dda 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -6,6 +6,13 @@ /out-tsc server.run.js +# docker +Dockerfile +entrypoint.sh +nginx-mempool.conf +nginx.conf +wait-for + # Only exists if Bazel was run /bazel-out diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 378ef11ed..6b7ec7f51 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -22,6 +22,7 @@ import { FiatCurrencyPipe } from './shared/pipes/fiat-currency.pipe'; import { ShortenStringPipe } from './shared/pipes/shorten-string-pipe/shorten-string.pipe'; import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe'; import { AppPreloadingStrategy } from './app.preloading-strategy'; +import { ServicesApiServices } from './services/services-api.service'; const providers = [ ElectrsApiService, @@ -40,6 +41,7 @@ const providers = [ FiatCurrencyPipe, CapAddressPipe, AppPreloadingStrategy, + ServicesApiServices, { provide: HTTP_INTERCEPTORS, useClass: HttpCacheInterceptor, multi: true } ]; diff --git a/frontend/src/app/components/about/about-sponsors.component.html b/frontend/src/app/components/about/about-sponsors.component.html index f76d74f8b..9471fc78f 100644 --- a/frontend/src/app/components/about/about-sponsors.component.html +++ b/frontend/src/app/components/about/about-sponsors.component.html @@ -1,16 +1,16 @@ -
+

If you're an individual...

- Become a Community Sponsor + Become a Community Sponsor

If you're a business...

- Become an Enterprise Sponsor + Become an Enterprise Sponsor
-
\ No newline at end of file +
diff --git a/frontend/src/app/components/about/about-sponsors.component.scss b/frontend/src/app/components/about/about-sponsors.component.scss index f3e675fd4..7c01bb9a3 100644 --- a/frontend/src/app/components/about/about-sponsors.component.scss +++ b/frontend/src/app/components/about/about-sponsors.component.scss @@ -6,6 +6,11 @@ align-items: center; gap: 20px; margin: 68px auto; + text-align: center; +} + +#become-sponsor-container.account { + margin: 20px auto; } .become-sponsor { diff --git a/frontend/src/app/components/about/about-sponsors.component.ts b/frontend/src/app/components/about/about-sponsors.component.ts index 31863cd8f..6a47c3bd4 100644 --- a/frontend/src/app/components/about/about-sponsors.component.ts +++ b/frontend/src/app/components/about/about-sponsors.component.ts @@ -1,4 +1,4 @@ -import { Component } from '@angular/core'; +import { Component, Input } from '@angular/core'; import { EnterpriseService } from '../../services/enterprise.service'; @Component({ @@ -7,6 +7,9 @@ import { EnterpriseService } from '../../services/enterprise.service'; styleUrls: ['./about-sponsors.component.scss'], }) export class AboutSponsorsComponent { + @Input() host = 'https://mempool.space'; + @Input() context = 'about'; + constructor(private enterpriseService: EnterpriseService) { } diff --git a/frontend/src/app/components/accelerate-preview/accelerate-preview.component.ts b/frontend/src/app/components/accelerate-preview/accelerate-preview.component.ts index d8ebb3830..43ccce3b7 100644 --- a/frontend/src/app/components/accelerate-preview/accelerate-preview.component.ts +++ b/frontend/src/app/components/accelerate-preview/accelerate-preview.component.ts @@ -4,6 +4,7 @@ import { Subscription, catchError, of, tap } from 'rxjs'; import { StorageService } from '../../services/storage.service'; import { Transaction } from '../../interfaces/electrs.interface'; import { nextRoundNumber } from '../../shared/common.utils'; +import { ServicesApiServices } from '../../services/services-api.service'; import { AudioService } from '../../services/audio.service'; export type AccelerationEstimate = { @@ -62,7 +63,7 @@ export class AcceleratePreviewComponent implements OnInit, OnDestroy, OnChanges maxRateOptions: RateOption[] = []; constructor( - private apiService: ApiService, + private servicesApiService: ServicesApiServices, private storageService: StorageService, private audioService: AudioService, private cd: ChangeDetectorRef @@ -83,7 +84,7 @@ export class AcceleratePreviewComponent implements OnInit, OnDestroy, OnChanges ngOnInit() { this.user = this.storageService.getAuth()?.user ?? null; - this.estimateSubscription = this.apiService.estimate$(this.tx.txid).pipe( + this.estimateSubscription = this.servicesApiService.estimate$(this.tx.txid).pipe( tap((response) => { if (response.status === 204) { this.estimate = undefined; @@ -183,7 +184,7 @@ export class AcceleratePreviewComponent implements OnInit, OnDestroy, OnChanges if (this.accelerationSubscription) { this.accelerationSubscription.unsubscribe(); } - this.accelerationSubscription = this.apiService.accelerate$( + this.accelerationSubscription = this.servicesApiService.accelerate$( this.tx.txid, this.userBid ).subscribe({ @@ -213,4 +214,4 @@ export class AcceleratePreviewComponent implements OnInit, OnDestroy, OnChanges onResize(): void { this.isMobile = window.innerWidth <= 767.98; } -} \ No newline at end of file +} diff --git a/frontend/src/app/components/acceleration/acceleration-fees-graph/acceleration-fees-graph.component.html b/frontend/src/app/components/acceleration/acceleration-fees-graph/acceleration-fees-graph.component.html index 98095aa07..3698a3060 100644 --- a/frontend/src/app/components/acceleration/acceleration-fees-graph/acceleration-fees-graph.component.html +++ b/frontend/src/app/components/acceleration/acceleration-fees-graph/acceleration-fees-graph.component.html @@ -27,12 +27,6 @@ -
-
-
Total Bid Boost
-
-
-
diff --git a/frontend/src/app/components/acceleration/acceleration-fees-graph/acceleration-fees-graph.component.scss b/frontend/src/app/components/acceleration/acceleration-fees-graph/acceleration-fees-graph.component.scss index c4b4335ee..e01beb350 100644 --- a/frontend/src/app/components/acceleration/acceleration-fees-graph/acceleration-fees-graph.component.scss +++ b/frontend/src/app/components/acceleration/acceleration-fees-graph/acceleration-fees-graph.component.scss @@ -53,11 +53,6 @@ padding-bottom: 55px; } } -.chart-widget { - width: 100%; - height: 100%; - max-height: 290px; -} h5 { margin-bottom: 10px; diff --git a/frontend/src/app/components/acceleration/acceleration-fees-graph/acceleration-fees-graph.component.ts b/frontend/src/app/components/acceleration/acceleration-fees-graph/acceleration-fees-graph.component.ts index d75cbba2d..e786635ba 100644 --- a/frontend/src/app/components/acceleration/acceleration-fees-graph/acceleration-fees-graph.component.ts +++ b/frontend/src/app/components/acceleration/acceleration-fees-graph/acceleration-fees-graph.component.ts @@ -1,8 +1,7 @@ import { ChangeDetectionStrategy, ChangeDetectorRef, Component, Inject, Input, LOCALE_ID, OnDestroy, OnInit } from '@angular/core'; import { EChartsOption, graphic } from 'echarts'; -import { Observable, Subscription, combineLatest } from 'rxjs'; +import { Observable, Subscription, combineLatest, fromEvent } from 'rxjs'; import { map, max, startWith, switchMap, tap } from 'rxjs/operators'; -import { ApiService } from '../../../services/api.service'; import { SeoService } from '../../../services/seo.service'; import { formatNumber } from '@angular/common'; import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms'; @@ -11,6 +10,8 @@ import { StorageService } from '../../../services/storage.service'; import { MiningService } from '../../../services/mining.service'; import { ActivatedRoute } from '@angular/router'; import { Acceleration } from '../../../interfaces/node-api.interface'; +import { ServicesApiServices } from '../../../services/services-api.service'; +import { ApiService } from '../../../services/api.service'; @Component({ selector: 'app-acceleration-fees-graph', @@ -28,6 +29,7 @@ import { Acceleration } from '../../../interfaces/node-api.interface'; }) export class AccelerationFeesGraphComponent implements OnInit, OnDestroy { @Input() widget: boolean = false; + @Input() height: number | string = '200'; @Input() right: number | string = 45; @Input() left: number | string = 75; @Input() accelerations$: Observable; @@ -54,6 +56,7 @@ export class AccelerationFeesGraphComponent implements OnInit, OnDestroy { @Inject(LOCALE_ID) public locale: string, private seoService: SeoService, private apiService: ApiService, + private servicesApiService: ServicesApiServices, private formBuilder: UntypedFormBuilder, private storageService: StorageService, private miningService: MiningService, @@ -72,8 +75,9 @@ export class AccelerationFeesGraphComponent implements OnInit, OnDestroy { this.timespan = this.miningWindowPreference; this.statsObservable$ = combineLatest([ - (this.accelerations$ || this.apiService.getAccelerationHistory$({ timeframe: this.miningWindowPreference })), + (this.accelerations$ || this.servicesApiService.getAccelerationHistory$({ timeframe: this.miningWindowPreference })), this.apiService.getHistoricalBlockFees$(this.miningWindowPreference), + fromEvent(window, 'resize').pipe(startWith(null)), ]).pipe( tap(([accelerations, blockFeesResponse]) => { this.prepareChartOptions(accelerations, blockFeesResponse.body); @@ -101,7 +105,7 @@ export class AccelerationFeesGraphComponent implements OnInit, OnDestroy { this.isLoading = true; this.storageService.setValue('miningWindowPreference', timespan); this.timespan = timespan; - return this.apiService.getAccelerationHistory$({}); + return this.servicesApiService.getAccelerationHistory$({}); }) ), this.radioGroupForm.get('dateSpan').valueChanges.pipe( @@ -173,6 +177,7 @@ export class AccelerationFeesGraphComponent implements OnInit, OnDestroy { ], animation: false, grid: { + height: this.height, right: this.right, left: this.left, bottom: this.widget ? 30 : 80, diff --git a/frontend/src/app/components/acceleration/accelerations-list/accelerations-list.component.ts b/frontend/src/app/components/acceleration/accelerations-list/accelerations-list.component.ts index 69af8b966..c1ab011ea 100644 --- a/frontend/src/app/components/acceleration/accelerations-list/accelerations-list.component.ts +++ b/frontend/src/app/components/acceleration/accelerations-list/accelerations-list.component.ts @@ -1,9 +1,9 @@ import { Component, OnInit, ChangeDetectionStrategy, Input, ChangeDetectorRef } from '@angular/core'; import { Observable, catchError, of, switchMap, tap } from 'rxjs'; import { Acceleration, BlockExtended } from '../../../interfaces/node-api.interface'; -import { ApiService } from '../../../services/api.service'; import { StateService } from '../../../services/state.service'; import { WebsocketService } from '../../../services/websocket.service'; +import { ServicesApiServices } from '../../../services/services-api.service'; @Component({ selector: 'app-accelerations-list', @@ -26,7 +26,7 @@ export class AccelerationsListComponent implements OnInit { skeletonLines: number[] = []; constructor( - private apiService: ApiService, + private servicesApiService: ServicesApiServices, private websocketService: WebsocketService, public stateService: StateService, private cd: ChangeDetectorRef, @@ -41,7 +41,7 @@ export class AccelerationsListComponent implements OnInit { this.skeletonLines = this.widget === true ? [...Array(6).keys()] : [...Array(15).keys()]; this.paginationMaxSize = window.matchMedia('(max-width: 670px)').matches ? 3 : 5; - const accelerationObservable$ = this.accelerations$ || (this.pending ? this.apiService.getAccelerations$() : this.apiService.getAccelerationHistory$({ timeframe: '1m' })); + const accelerationObservable$ = this.accelerations$ || (this.pending ? this.servicesApiService.getAccelerations$() : this.servicesApiService.getAccelerationHistory$({ timeframe: '1m' })); this.accelerationList$ = accelerationObservable$.pipe( switchMap(accelerations => { if (this.pending) { diff --git a/frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.html b/frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.html index 19d01e726..6d9e49265 100644 --- a/frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.html +++ b/frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.html @@ -37,6 +37,11 @@
+ +
Mempool Goggles: Accelerations
+   + +
@@ -48,7 +53,15 @@
- +
Total Bid Boost
+
+ +
diff --git a/frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.scss b/frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.scss index 145569342..c8755c94e 100644 --- a/frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.scss +++ b/frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.scss @@ -17,6 +17,16 @@ } } +.mempool-graph { + height: 295px; + @media (min-width: 768px) { + height: 325px; + } + @media (min-width: 992px) { + height: 409px; + } +} + .card-title { font-size: 1rem; color: #4a68b9; @@ -135,7 +145,12 @@ } .card { - height: 385px; + @media (min-width: 768px) { + height: 420px; + } + @media (min-width: 992px) { + height: 510px; + } } .list-card { height: 410px; @@ -145,7 +160,16 @@ } .mempool-block-wrapper { - max-height: 380px; - max-width: 380px; + max-height: 430px; + max-width: 430px; margin: auto; + + @media (min-width: 768px) { + max-height: 344px; + max-width: 344px; + } + @media (min-width: 992px) { + max-height: 430px; + max-width: 430px; + } } \ No newline at end of file diff --git a/frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.ts b/frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.ts index 79a77a600..a2abc657a 100644 --- a/frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.ts +++ b/frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.ts @@ -1,14 +1,14 @@ -import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; +import { ChangeDetectionStrategy, Component, HostListener, OnInit } from '@angular/core'; import { SeoService } from '../../../services/seo.service'; import { WebsocketService } from '../../../services/websocket.service'; import { Acceleration, BlockExtended } from '../../../interfaces/node-api.interface'; import { StateService } from '../../../services/state.service'; -import { Observable, Subject, catchError, combineLatest, distinctUntilChanged, interval, map, of, share, startWith, switchMap, tap } from 'rxjs'; -import { ApiService } from '../../../services/api.service'; +import { Observable, catchError, combineLatest, distinctUntilChanged, interval, map, of, share, startWith, switchMap, tap } from 'rxjs'; import { Color } from '../../block-overview-graph/sprite-types'; import { hexToColor } from '../../block-overview-graph/utils'; import TxView from '../../block-overview-graph/tx-view'; import { feeLevels, mempoolFeeColors } from '../../../app.constants'; +import { ServicesApiServices } from '../../../services/services-api.service'; const acceleratedColor: Color = hexToColor('8F5FF6'); const normalColors = mempoolFeeColors.map(hex => hexToColor(hex + '5F')); @@ -30,43 +30,48 @@ export class AcceleratorDashboardComponent implements OnInit { minedAccelerations$: Observable; loadingBlocks: boolean = true; + graphHeight: number = 300; + constructor( private seoService: SeoService, private websocketService: WebsocketService, - private apiService: ApiService, + private serviceApiServices: ServicesApiServices, private stateService: StateService, ) { this.seoService.setTitle($localize`:@@a681a4e2011bb28157689dbaa387de0dd0aa0c11:Accelerator Dashboard`); } ngOnInit(): void { + this.onResize(); this.websocketService.want(['blocks', 'mempool-blocks', 'stats']); this.pendingAccelerations$ = interval(30000).pipe( startWith(true), switchMap(() => { - return this.apiService.getAccelerations$(); - }), - catchError((e) => { - return of([]); + return this.serviceApiServices.getAccelerations$().pipe( + catchError(() => { + return of([]); + }), + ); }), share(), ); this.accelerations$ = this.stateService.chainTip$.pipe( distinctUntilChanged(), - switchMap((chainTip) => { - return this.apiService.getAccelerationHistory$({ timeframe: '1m' }); - }), - catchError((e) => { - return of([]); + switchMap(() => { + return this.serviceApiServices.getAccelerationHistory$({ timeframe: '1m' }).pipe( + catchError(() => { + return of([]); + }), + ); }), share(), ); this.minedAccelerations$ = this.accelerations$.pipe( map(accelerations => { - return accelerations.filter(acc => ['mined', 'completed'].includes(acc.status)) + return accelerations.filter(acc => ['mined', 'completed', 'failed'].includes(acc.status)); }) ); @@ -119,4 +124,15 @@ export class AcceleratorDashboardComponent implements OnInit { return normalColors[feeLevelIndex] || normalColors[mempoolFeeColors.length - 1]; } } + + @HostListener('window:resize', ['$event']) + onResize(): void { + if (window.innerWidth >= 992) { + this.graphHeight = 330; + } else if (window.innerWidth >= 768) { + this.graphHeight = 245; + } else { + this.graphHeight = 210; + } + } } diff --git a/frontend/src/app/components/acceleration/pending-stats/pending-stats.component.ts b/frontend/src/app/components/acceleration/pending-stats/pending-stats.component.ts index f344c37a0..ed7061156 100644 --- a/frontend/src/app/components/acceleration/pending-stats/pending-stats.component.ts +++ b/frontend/src/app/components/acceleration/pending-stats/pending-stats.component.ts @@ -1,8 +1,8 @@ import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core'; import { Observable, of } from 'rxjs'; import { switchMap } from 'rxjs/operators'; -import { ApiService } from '../../../services/api.service'; import { Acceleration } from '../../../interfaces/node-api.interface'; +import { ServicesApiServices } from '../../../services/services-api.service'; @Component({ selector: 'app-pending-stats', @@ -15,11 +15,11 @@ export class PendingStatsComponent implements OnInit { public accelerationStats$: Observable; constructor( - private apiService: ApiService, + private servicesApiService: ServicesApiServices, ) { } ngOnInit(): void { - this.accelerationStats$ = (this.accelerations$ || this.apiService.getAccelerations$()).pipe( + this.accelerationStats$ = (this.accelerations$ || this.servicesApiService.getAccelerations$()).pipe( switchMap(accelerations => { let totalAccelerations = 0; let totalFeeDelta = 0; diff --git a/frontend/src/app/components/address-labels/address-labels.component.ts b/frontend/src/app/components/address-labels/address-labels.component.ts index f4b3d0ca5..2365c167f 100644 --- a/frontend/src/app/components/address-labels/address-labels.component.ts +++ b/frontend/src/app/components/address-labels/address-labels.component.ts @@ -43,7 +43,7 @@ export class AddressLabelsComponent implements OnChanges { handleVin() { if (this.vin.inner_witnessscript_asm) { - if (this.vin.inner_witnessscript_asm.indexOf('OP_DEPTH OP_PUSHNUM_12 OP_EQUAL OP_IF OP_PUSHNUM_11') === 0) { + if (this.vin.inner_witnessscript_asm.indexOf('OP_DEPTH OP_PUSHNUM_12 OP_EQUAL OP_IF OP_PUSHNUM_11') === 0 || this.vin.inner_witnessscript_asm.indexOf('OP_PUSHNUM_15 OP_CHECKMULTISIG OP_IFDUP OP_NOTIF OP_PUSHBYTES_2') === 1259) { if (this.vin.witness.length > 11) { this.label = 'Liquid Peg Out'; } else { diff --git a/frontend/src/app/components/address/address.component.ts b/frontend/src/app/components/address/address.component.ts index 0e10b207f..7e613dba6 100644 --- a/frontend/src/app/components/address/address.component.ts +++ b/frontend/src/app/components/address/address.component.ts @@ -31,8 +31,7 @@ export class AddressComponent implements OnInit, OnDestroy { addressLoadingStatus$: Observable; addressInfo: null | AddressInformation = null; - totalConfirmedTxCount = 0; - loadedConfirmedTxCount = 0; + fullyLoaded = false; txCount = 0; received = 0; sent = 0; @@ -66,7 +65,7 @@ export class AddressComponent implements OnInit, OnDestroy { switchMap((params: ParamMap) => { this.error = undefined; this.isLoadingAddress = true; - this.loadedConfirmedTxCount = 0; + this.fullyLoaded = false; this.address = null; this.isLoadingTransactions = true; this.transactions = null; @@ -105,7 +104,7 @@ export class AddressComponent implements OnInit, OnDestroy { .pipe( filter((address) => !!address), tap((address: Address) => { - if ((this.stateService.network === 'liquid' || this.stateService.network === 'liquidtestnet') && /^([m-zA-HJ-NP-Z1-9]{26,35}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[a-km-zA-HJ-NP-Z1-9]{80})$/.test(address.address)) { + if ((this.stateService.network === 'liquid' || this.stateService.network === 'liquidtestnet') && /^([a-zA-HJ-NP-Z1-9]{26,35}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[a-km-zA-HJ-NP-Z1-9]{80})$/.test(address.address)) { this.apiService.validateAddress$(address.address) .subscribe((addressInfo) => { this.addressInfo = addressInfo; @@ -128,7 +127,6 @@ export class AddressComponent implements OnInit, OnDestroy { this.tempTransactions = transactions; if (transactions.length) { this.lastTransactionTxId = transactions[transactions.length - 1].txid; - this.loadedConfirmedTxCount += transactions.filter((tx) => tx.status.confirmed).length; } const fetchTxs: string[] = []; @@ -191,8 +189,6 @@ export class AddressComponent implements OnInit, OnDestroy { this.audioService.playSound('magic'); } } - this.totalConfirmedTxCount++; - this.loadedConfirmedTxCount++; }); } @@ -252,16 +248,19 @@ export class AddressComponent implements OnInit, OnDestroy { } loadMore() { - if (this.isLoadingTransactions || !this.totalConfirmedTxCount || this.loadedConfirmedTxCount >= this.totalConfirmedTxCount) { + if (this.isLoadingTransactions || this.fullyLoaded) { return; } this.isLoadingTransactions = true; this.retryLoadMore = false; this.electrsApiService.getAddressTransactions$(this.address.address, this.lastTransactionTxId) .subscribe((transactions: Transaction[]) => { - this.lastTransactionTxId = transactions[transactions.length - 1].txid; - this.loadedConfirmedTxCount += transactions.length; - this.transactions = this.transactions.concat(transactions); + if (transactions && transactions.length) { + this.lastTransactionTxId = transactions[transactions.length - 1].txid; + this.transactions = this.transactions.concat(transactions); + } else { + this.fullyLoaded = true; + } this.isLoadingTransactions = false; }, (error) => { @@ -278,7 +277,6 @@ export class AddressComponent implements OnInit, OnDestroy { this.received = this.address.chain_stats.funded_txo_sum + this.address.mempool_stats.funded_txo_sum; this.sent = this.address.chain_stats.spent_txo_sum + this.address.mempool_stats.spent_txo_sum; this.txCount = this.address.chain_stats.tx_count + this.address.mempool_stats.tx_count; - this.totalConfirmedTxCount = this.address.chain_stats.tx_count; } ngOnDestroy() { diff --git a/frontend/src/app/components/amount/amount.component.html b/frontend/src/app/components/amount/amount.component.html index 29f61ca41..34f9be8ae 100644 --- a/frontend/src/app/components/amount/amount.component.html +++ b/frontend/src/app/components/amount/amount.component.html @@ -19,7 +19,7 @@ ‎{{ addPlus && satoshis >= 0 ? '+' : '' }}{{ satoshis / 100000000 | number : digitsInfo }} - L- + L- tL- t sBTC diff --git a/frontend/src/app/components/amount/amount.component.ts b/frontend/src/app/components/amount/amount.component.ts index 479ae4791..9c779265c 100644 --- a/frontend/src/app/components/amount/amount.component.ts +++ b/frontend/src/app/components/amount/amount.component.ts @@ -23,6 +23,7 @@ export class AmountComponent implements OnInit, OnDestroy { @Input() noFiat = false; @Input() addPlus = false; @Input() blockConversion: Price; + @Input() forceBtc: boolean = false; constructor( private stateService: StateService, diff --git a/frontend/src/app/components/block-filters/block-filters.component.html b/frontend/src/app/components/block-filters/block-filters.component.html index f60b04cdd..8c79cd438 100644 --- a/frontend/src/app/components/block-filters/block-filters.component.html +++ b/frontend/src/app/components/block-filters/block-filters.component.html @@ -1,4 +1,4 @@ -
+
Match
+
+ + +
{{ group.label }}
diff --git a/frontend/src/app/components/block-filters/block-filters.component.scss b/frontend/src/app/components/block-filters/block-filters.component.scss index 6406a1d93..1009efd72 100644 --- a/frontend/src/app/components/block-filters/block-filters.component.scss +++ b/frontend/src/app/components/block-filters/block-filters.component.scss @@ -77,6 +77,49 @@ } } + &.any-mode { + .filter-tag { + border: solid 1px #1a9436; + &.selected { + background-color: #1a9436; + } + } + } + + .btn-group { + font-size: 0.9em; + margin-right: 0.25em; + } + + .mode-toggle { + padding: 0.2em 0.5em; + pointer-events: all; + line-height: 1.5; + background: #181b2daf; + + &:first-child { + border-top-left-radius: 0.2rem; + border-bottom-left-radius: 0.2rem; + } + &:last-child { + border-top-right-radius: 0.2rem; + border-bottom-right-radius: 0.2rem; + } + + &.blue { + border: solid 1px #105fb0; + &.active { + background: #105fb0; + } + } + &.green { + border: solid 1px #1a9436; + &.active { + background: #1a9436; + } + } + } + :host-context(.block-overview-graph:hover) &, &:hover, &:active { .menu-toggle { opacity: 0.5; @@ -132,6 +175,11 @@ .filter-tag { font-size: 0.7em; } + .mode-toggle { + font-size: 0.7em; + margin-bottom: 5px; + margin-top: 2px; + } } &.tiny { diff --git a/frontend/src/app/components/block-filters/block-filters.component.ts b/frontend/src/app/components/block-filters/block-filters.component.ts index 9951984df..a16475c23 100644 --- a/frontend/src/app/components/block-filters/block-filters.component.ts +++ b/frontend/src/app/components/block-filters/block-filters.component.ts @@ -1,5 +1,5 @@ import { Component, EventEmitter, Output, HostListener, Input, ChangeDetectorRef, OnChanges, SimpleChanges, OnInit, OnDestroy } from '@angular/core'; -import { FilterGroups, TransactionFilters } from '../../shared/filters.utils'; +import { ActiveFilter, FilterGroups, FilterMode, TransactionFilters } from '../../shared/filters.utils'; import { StateService } from '../../services/state.service'; import { Subscription } from 'rxjs'; @@ -12,7 +12,7 @@ import { Subscription } from 'rxjs'; export class BlockFiltersComponent implements OnInit, OnChanges, OnDestroy { @Input() cssWidth: number = 800; @Input() excludeFilters: string[] = []; - @Output() onFilterChanged: EventEmitter = new EventEmitter(); + @Output() onFilterChanged: EventEmitter = new EventEmitter(); filterSubscription: Subscription; @@ -21,6 +21,7 @@ export class BlockFiltersComponent implements OnInit, OnChanges, OnDestroy { disabledFilters: { [key: string]: boolean } = {}; activeFilters: string[] = []; filterFlags: { [key: string]: boolean } = {}; + filterMode: FilterMode = 'and'; menuOpen: boolean = false; constructor( @@ -29,15 +30,16 @@ export class BlockFiltersComponent implements OnInit, OnChanges, OnDestroy { ) {} ngOnInit(): void { - this.filterSubscription = this.stateService.activeGoggles$.subscribe((activeFilters: string[]) => { + this.filterSubscription = this.stateService.activeGoggles$.subscribe((active: ActiveFilter) => { + this.filterMode = active.mode; for (const key of Object.keys(this.filterFlags)) { this.filterFlags[key] = false; } - for (const key of activeFilters) { + for (const key of active.filters) { this.filterFlags[key] = !this.disabledFilters[key]; } - this.activeFilters = [...activeFilters.filter(key => !this.disabledFilters[key])]; - this.onFilterChanged.emit(this.getBooleanFlags()); + this.activeFilters = [...active.filters.filter(key => !this.disabledFilters[key])]; + this.onFilterChanged.emit({ mode: active.mode, filters: this.activeFilters }); }); } @@ -53,6 +55,12 @@ export class BlockFiltersComponent implements OnInit, OnChanges, OnDestroy { } } + setFilterMode(mode): void { + this.filterMode = mode; + this.onFilterChanged.emit({ mode: this.filterMode, filters: this.activeFilters }); + this.stateService.activeGoggles$.next({ mode: this.filterMode, filters: [...this.activeFilters] }); + } + toggleFilter(key): void { const filter = this.filters[key]; this.filterFlags[key] = !this.filterFlags[key]; @@ -73,8 +81,8 @@ export class BlockFiltersComponent implements OnInit, OnChanges, OnDestroy { this.activeFilters = this.activeFilters.filter(f => f != key); } const booleanFlags = this.getBooleanFlags(); - this.onFilterChanged.emit(booleanFlags); - this.stateService.activeGoggles$.next([...this.activeFilters]); + this.onFilterChanged.emit({ mode: this.filterMode, filters: this.activeFilters }); + this.stateService.activeGoggles$.next({ mode: this.filterMode, filters: [...this.activeFilters] }); } getBooleanFlags(): bigint | null { @@ -90,7 +98,7 @@ export class BlockFiltersComponent implements OnInit, OnChanges, OnDestroy { @HostListener('document:click', ['$event']) onClick(event): boolean { // click away from menu - if (!event.target.closest('button')) { + if (!event.target.closest('button') && !event.target.closest('label')) { this.menuOpen = false; } return true; diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.html b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.html index 9d27d8d90..34d192678 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.html +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.html @@ -13,6 +13,9 @@ [auditEnabled]="auditHighlighting" [blockConversion]="blockConversion" > - + +
+ Your browser does not support this feature. +
diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.scss b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.scss index d30dd3305..92964d948 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.scss +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.scss @@ -7,6 +7,19 @@ justify-content: center; align-items: center; grid-column: 1/-1; + + .placeholder { + display: flex; + position: absolute; + left: 0; + right: 0; + top: 0; + bottom: 0; + height: 100%; + width: 100%; + align-items: center; + justify-content: center; + } } .grid-align { diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index d6000e27b..95305d72f 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -9,6 +9,8 @@ import { Price } from '../../services/price.service'; import { StateService } from '../../services/state.service'; import { Subscription } from 'rxjs'; import { defaultColorFunction, setOpacity, defaultFeeColors, defaultAuditFeeColors, defaultMarginalFeeColors, defaultAuditColors } from './utils'; +import { ActiveFilter, FilterMode, toFlags } from '../../shared/filters.utils'; +import { detectWebGL } from '../../shared/graphs.utils'; const unmatchedOpacity = 0.2; const unmatchedFeeColors = defaultFeeColors.map(c => setOpacity(c, unmatchedOpacity)); @@ -42,6 +44,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On @Input() showFilters: boolean = false; @Input() excludeFilters: string[] = []; @Input() filterFlags: bigint | null = null; + @Input() filterMode: FilterMode = 'and'; @Input() blockConversion: Price; @Input() overrideColors: ((tx: TxView) => Color) | null = null; @Output() txClickEvent = new EventEmitter<{ tx: TransactionStripped, keyModifier: boolean}>(); @@ -75,11 +78,14 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On filtersAvailable: boolean = true; activeFilterFlags: bigint | null = null; + webGlEnabled = true; + constructor( readonly ngZone: NgZone, readonly elRef: ElementRef, private stateService: StateService, ) { + this.webGlEnabled = detectWebGL(); this.vertexArray = new FastVertexArray(512, TxSprite.dataSize); this.searchSubscription = this.stateService.searchText$.subscribe((text) => { this.searchText = text; @@ -113,16 +119,17 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On if (changes.overrideColor && this.scene) { this.scene.setColorFunction(this.overrideColors); } - if ((changes.filterFlags || changes.showFilters)) { + if ((changes.filterFlags || changes.showFilters || changes.filterMode)) { this.setFilterFlags(); } } - setFilterFlags(flags?: bigint | null): void { - this.activeFilterFlags = this.filterFlags || flags || null; + setFilterFlags(goggle?: ActiveFilter): void { + this.filterMode = goggle?.mode || this.filterMode; + this.activeFilterFlags = goggle?.filters ? toFlags(goggle.filters) : this.filterFlags; if (this.scene) { - if (flags != null) { - this.scene.setColorFunction(this.getFilterColorFunction(flags)); + if (this.activeFilterFlags != null && this.filtersAvailable) { + this.scene.setColorFunction(this.getFilterColorFunction(this.activeFilterFlags)); } else { this.scene.setColorFunction(this.overrideColors); } @@ -156,7 +163,11 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On // initialize the scene without any entry transition setup(transactions: TransactionStripped[]): void { - this.filtersAvailable = transactions.reduce((flagSet, tx) => flagSet || tx.flags > 0, false); + const filtersAvailable = transactions.reduce((flagSet, tx) => flagSet || tx.flags > 0, false); + if (filtersAvailable !== this.filtersAvailable) { + this.setFilterFlags(); + } + this.filtersAvailable = filtersAvailable; if (this.scene) { this.scene.setup(transactions); this.readyNextFrame = true; @@ -499,11 +510,13 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On } onTxClick(cssX: number, cssY: number, keyModifier: boolean = false) { - const x = cssX * window.devicePixelRatio; - const y = cssY * window.devicePixelRatio; - const selected = this.scene.getTxAt({ x, y }); - if (selected && selected.txid) { - this.txClickEvent.emit({ tx: selected, keyModifier }); + if (this.scene) { + const x = cssX * window.devicePixelRatio; + const y = cssY * window.devicePixelRatio; + const selected = this.scene.getTxAt({ x, y }); + if (selected && selected.txid) { + this.txClickEvent.emit({ tx: selected, keyModifier }); + } } } @@ -523,7 +536,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On getFilterColorFunction(flags: bigint): ((tx: TxView) => Color) { return (tx: TxView) => { - if ((tx.bigintFlags & flags) === flags) { + if ((this.filterMode === 'and' && (tx.bigintFlags & flags) === flags) || (this.filterMode === 'or' && (flags === 0n || (tx.bigintFlags & flags) > 0n))) { return defaultColorFunction(tx); } else { return defaultColorFunction( diff --git a/frontend/src/app/components/block/block-preview.component.ts b/frontend/src/app/components/block/block-preview.component.ts index e634ae11f..3e1d9b409 100644 --- a/frontend/src/app/components/block/block-preview.component.ts +++ b/frontend/src/app/components/block/block-preview.component.ts @@ -10,6 +10,7 @@ import { BlockExtended, TransactionStripped } from '../../interfaces/node-api.in import { ApiService } from '../../services/api.service'; import { seoDescriptionNetwork } from '../../shared/common.utils'; import { BlockOverviewGraphComponent } from '../../components/block-overview-graph/block-overview-graph.component'; +import { ServicesApiServices } from '../../services/services-api.service'; @Component({ selector: 'app-block-preview', @@ -42,7 +43,8 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { public stateService: StateService, private seoService: SeoService, private openGraphService: OpenGraphService, - private apiService: ApiService + private apiService: ApiService, + private servicesApiService: ServicesApiServices, ) { } ngOnInit() { @@ -134,7 +136,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { return of(transactions); }) ), - this.stateService.env.ACCELERATOR === true && block.height > 819500 ? this.apiService.getAccelerationHistory$({ blockHash: block.id }) : of([]) + this.stateService.env.ACCELERATOR === true && block.height > 819500 ? this.servicesApiService.getAccelerationHistory$({ blockHash: block.id }) : of([]) ]); } ), diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 6a995127b..5bba24852 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -16,6 +16,7 @@ import { detectWebGL } from '../../shared/graphs.utils'; import { seoDescriptionNetwork } from '../../shared/common.utils'; import { PriceService, Price } from '../../services/price.service'; import { CacheService } from '../../services/cache.service'; +import { ServicesApiServices } from '../../services/services-api.service'; @Component({ selector: 'app-block', @@ -103,6 +104,7 @@ export class BlockComponent implements OnInit, OnDestroy { private apiService: ApiService, private priceService: PriceService, private cacheService: CacheService, + private servicesApiService: ServicesApiServices, ) { this.webGlEnabled = detectWebGL(); } @@ -329,7 +331,7 @@ export class BlockComponent implements OnInit, OnDestroy { return of(null); }) ), - this.stateService.env.ACCELERATOR === true && block.height > 819500 ? this.apiService.getAccelerationHistory$({ blockHash: block.id }) : of([]) + this.stateService.env.ACCELERATOR === true && block.height > 819500 ? this.servicesApiService.getAccelerationHistory$({ blockHash: block.id }) : of([]) ]); }) ) diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html index 680beb006..443fc1946 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html @@ -26,7 +26,7 @@
-   +
-   +
- + - - - - + diff --git a/frontend/src/app/components/difficulty-mining/difficulty-mining.component.html b/frontend/src/app/components/difficulty-mining/difficulty-mining.component.html index 9f3706916..215b5c68a 100644 --- a/frontend/src/app/components/difficulty-mining/difficulty-mining.component.html +++ b/frontend/src/app/components/difficulty-mining/difficulty-mining.component.html @@ -14,7 +14,7 @@
Estimate
-
+
@@ -24,9 +24,6 @@ {{ epochData.change | absolute | number: '1.2-2' }} %
- -
-
Previous: @@ -49,13 +46,15 @@
Next Halving
- {{ timeUntilHalving | date }} + + {{ i }} blocks + {{ i }} block
- +
- + {{ timeUntilHalving | date }}
diff --git a/frontend/src/app/components/difficulty-mining/difficulty-mining.component.ts b/frontend/src/app/components/difficulty-mining/difficulty-mining.component.ts index c1283b8b1..c650e2c02 100644 --- a/frontend/src/app/components/difficulty-mining/difficulty-mining.component.ts +++ b/frontend/src/app/components/difficulty-mining/difficulty-mining.component.ts @@ -16,6 +16,7 @@ interface EpochProgress { blocksUntilHalving: number; timeUntilHalving: number; timeAvg: number; + adjustedTimeAvg: number; } @Component({ @@ -85,6 +86,7 @@ export class DifficultyMiningComponent implements OnInit { blocksUntilHalving: this.blocksUntilHalving, timeUntilHalving: this.timeUntilHalving, timeAvg: da.timeAvg, + adjustedTimeAvg: da.adjustedTimeAvg, }; return data; }) diff --git a/frontend/src/app/components/difficulty/difficulty.component.html b/frontend/src/app/components/difficulty/difficulty.component.html index f08ea06f5..8011c7e6f 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.html +++ b/frontend/src/app/components/difficulty/difficulty.component.html @@ -42,7 +42,7 @@
Average block time
-
+
@@ -52,9 +52,6 @@ {{ epochData.change | absolute | number: '1.2-2' }} %
- -
-
Previous: diff --git a/frontend/src/app/components/difficulty/difficulty.component.ts b/frontend/src/app/components/difficulty/difficulty.component.ts index 81084f524..d37667312 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.ts +++ b/frontend/src/app/components/difficulty/difficulty.component.ts @@ -19,6 +19,7 @@ interface EpochProgress { blocksUntilHalving: number; timeUntilHalving: number; timeAvg: number; + adjustedTimeAvg: number; } type BlockStatus = 'mined' | 'behind' | 'ahead' | 'next' | 'remaining'; @@ -153,6 +154,7 @@ export class DifficultyComponent implements OnInit { blocksUntilHalving, timeUntilHalving, timeAvg: da.timeAvg, + adjustedTimeAvg: da.adjustedTimeAvg, }; return data; }) diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html index 83f8a3a4c..f3d340472 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html @@ -54,7 +54,7 @@
-
diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss index 886608573..32885d5de 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss @@ -57,8 +57,6 @@ } .chart-widget { width: 100%; - height: 100%; - height: 240px; } .pool-distribution { diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts index 9858807a6..2bda9a35a 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts @@ -1,6 +1,6 @@ import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit, HostBinding } from '@angular/core'; import { echarts, EChartsOption } from '../../graphs/echarts'; -import { merge, Observable, of } from 'rxjs'; +import { combineLatest, fromEvent, merge, Observable, of } from 'rxjs'; import { map, mergeMap, share, startWith, switchMap, tap } from 'rxjs/operators'; import { ApiService } from '../../services/api.service'; import { SeoService } from '../../services/seo.service'; @@ -31,6 +31,7 @@ import { seoDescriptionNetwork } from '../../shared/common.utils'; export class HashrateChartComponent implements OnInit { @Input() tableOnly = false; @Input() widget = false; + @Input() height: number = 300; @Input() right: number | string = 45; @Input() left: number | string = 75; @@ -86,28 +87,32 @@ export class HashrateChartComponent implements OnInit { } }); - this.hashrateObservable$ = merge( - this.radioGroupForm.get('dateSpan').valueChanges - .pipe( - startWith(this.radioGroupForm.controls.dateSpan.value), - switchMap((timespan) => { - if (!this.widget && !firstRun) { - this.storageService.setValue('miningWindowPreference', timespan); - } - this.timespan = timespan; - firstRun = false; - this.miningWindowPreference = timespan; - this.isLoading = true; - return this.apiService.getHistoricalHashrate$(this.timespan); - }) - ), - this.stateService.chainTip$ + this.hashrateObservable$ = combineLatest( + merge( + this.radioGroupForm.get('dateSpan').valueChanges .pipe( - switchMap(() => { + startWith(this.radioGroupForm.controls.dateSpan.value), + switchMap((timespan) => { + if (!this.widget && !firstRun) { + this.storageService.setValue('miningWindowPreference', timespan); + } + this.timespan = timespan; + firstRun = false; + this.miningWindowPreference = timespan; + this.isLoading = true; return this.apiService.getHistoricalHashrate$(this.timespan); }) - ) + ), + this.stateService.chainTip$ + .pipe( + switchMap(() => { + return this.apiService.getHistoricalHashrate$(this.timespan); + }) + ) + ), + fromEvent(window, 'resize').pipe(startWith(null)), ).pipe( + map(([response, _]) => response), tap((response: any) => { const data = response.body; @@ -221,6 +226,7 @@ export class HashrateChartComponent implements OnInit { ]), ], grid: { + height: (this.widget && this.height) ? this.height - 30 : undefined, top: this.widget ? 20 : 40, bottom: this.widget ? 30 : 70, right: this.right, diff --git a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts index b0557ca7c..3f0e9258f 100644 --- a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts +++ b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts @@ -11,6 +11,13 @@ import { MiningService } from '../../services/mining.service'; import { download } from '../../shared/graphs.utils'; import { ActivatedRoute } from '@angular/router'; +interface Hashrate { + timestamp: number; + avgHashRate: number; + share: number; + poolName: string; +} + @Component({ selector: 'app-hashrate-chart-pools', templateUrl: './hashrate-chart-pools.component.html', @@ -32,6 +39,7 @@ export class HashrateChartPoolsComponent implements OnInit { miningWindowPreference: string; radioGroupForm: UntypedFormGroup; + hashrates: Hashrate[]; chartOptions: EChartsOption = {}; chartInitOptions = { renderer: 'svg', @@ -87,56 +95,9 @@ export class HashrateChartPoolsComponent implements OnInit { return this.apiService.getHistoricalPoolsHashrate$(timespan) .pipe( tap((response) => { - const hashrates = response.body; + this.hashrates = response.body; // Prepare series (group all hashrates data point by pool) - const grouped = {}; - for (const hashrate of hashrates) { - if (!grouped.hasOwnProperty(hashrate.poolName)) { - grouped[hashrate.poolName] = []; - } - grouped[hashrate.poolName].push(hashrate); - } - - const series = []; - const legends = []; - for (const name in grouped) { - series.push({ - zlevel: 0, - stack: 'Total', - name: name, - showSymbol: false, - symbol: 'none', - data: grouped[name].map((val) => [val.timestamp * 1000, val.share * 100]), - type: 'line', - lineStyle: { width: 0 }, - areaStyle: { opacity: 1 }, - smooth: true, - color: poolsColor[name.replace(/[^a-zA-Z0-9]/g, '').toLowerCase()], - emphasis: { - disabled: true, - scale: false, - }, - }); - - legends.push({ - name: name, - inactiveColor: 'rgb(110, 112, 121)', - textStyle: { - color: 'white', - }, - icon: 'roundRect', - itemStyle: { - color: poolsColor[name.replace(/[^a-zA-Z0-9]/g, "").toLowerCase()], - }, - }); - } - - this.prepareChartOptions({ - legends: legends, - series: series, - }); - this.isLoading = false; - + const series = this.applyHashrates(); if (series.length === 0) { this.cd.markForCheck(); throw new Error(); @@ -156,6 +117,77 @@ export class HashrateChartPoolsComponent implements OnInit { ); } + applyHashrates(): any[] { + const times: { [time: number]: { hashrates: { [pool: string]: Hashrate } } } = {}; + const pools = {}; + for (const hashrate of this.hashrates) { + if (!times[hashrate.timestamp]) { + times[hashrate.timestamp] = { hashrates: {} }; + } + times[hashrate.timestamp].hashrates[hashrate.poolName] = hashrate; + if (!pools[hashrate.poolName]) { + pools[hashrate.poolName] = true; + } + } + + const sortedTimes = Object.keys(times).sort((a,b) => parseInt(a) - parseInt(b)).map(time => ({ time: parseInt(time), hashrates: times[time].hashrates })); + const lastHashrates = sortedTimes[sortedTimes.length - 1].hashrates; + const sortedPools = Object.keys(pools).sort((a,b) => { + if (lastHashrates[b]?.share ?? lastHashrates[a]?.share ?? false) { + // sort by descending share of hashrate in latest period + return (lastHashrates[b]?.share || 0) - (lastHashrates[a]?.share || 0); + } else { + // tiebreak by pool name + b < a; + } + }); + + const series = []; + const legends = []; + for (const name of sortedPools) { + const data = sortedTimes.map(({ time, hashrates }) => { + return [time * 1000, (hashrates[name]?.share || 0) * 100]; + }); + series.push({ + zlevel: 0, + stack: 'Total', + name: name, + showSymbol: false, + symbol: 'none', + data, + type: 'line', + lineStyle: { width: 0 }, + areaStyle: { opacity: 1 }, + smooth: true, + color: poolsColor[name.replace(/[^a-zA-Z0-9]/g, '').toLowerCase()], + emphasis: { + disabled: true, + scale: false, + }, + }); + + legends.push({ + name: name, + inactiveColor: 'rgb(110, 112, 121)', + textStyle: { + color: 'white', + }, + icon: 'roundRect', + itemStyle: { + color: poolsColor[name.replace(/[^a-zA-Z0-9]/g, "").toLowerCase()], + }, + }); + } + + this.prepareChartOptions({ + legends: legends, + series: series, + }); + this.isLoading = false; + + return series; + } + prepareChartOptions(data) { let title: object; if (data.series.length === 0) { @@ -256,6 +288,7 @@ export class HashrateChartPoolsComponent implements OnInit { }, }], }; + this.cd.markForCheck(); } onChartInit(ec) { diff --git a/frontend/src/app/components/lbtc-pegs-graph/lbtc-pegs-graph.component.ts b/frontend/src/app/components/lbtc-pegs-graph/lbtc-pegs-graph.component.ts index c4e8cbf91..9931fb78a 100644 --- a/frontend/src/app/components/lbtc-pegs-graph/lbtc-pegs-graph.component.ts +++ b/frontend/src/app/components/lbtc-pegs-graph/lbtc-pegs-graph.component.ts @@ -18,16 +18,15 @@ import { EChartsOption } from '../../graphs/echarts'; }) export class LbtcPegsGraphComponent implements OnInit, OnChanges { @Input() data: any; + @Input() height: number | string = '320'; pegsChartOptions: EChartsOption; - height: number | string = '200'; right: number | string = '10'; top: number | string = '20'; left: number | string = '50'; template: ('widget' | 'advanced') = 'widget'; isLoading = true; - pegsChartOption: EChartsOption = {}; pegsChartInitOption = { renderer: 'svg' }; @@ -41,20 +40,24 @@ export class LbtcPegsGraphComponent implements OnInit, OnChanges { } ngOnChanges() { - if (!this.data) { + if (!this.data?.liquidPegs) { return; } - this.pegsChartOptions = this.createChartOptions(this.data.series, this.data.labels); + if (!this.data.liquidReserves) { + this.pegsChartOptions = this.createChartOptions(this.data.liquidPegs.series, this.data.liquidPegs.labels); + } else { + this.pegsChartOptions = this.createChartOptions(this.data.liquidPegs.series, this.data.liquidPegs.labels, this.data.liquidReserves.series); + } } rendered() { - if (!this.data) { + if (!this.data.liquidPegs) { return; } this.isLoading = false; } - createChartOptions(series: number[], labels: string[]): EChartsOption { + createChartOptions(pegSeries: number[], labels: string[], reservesSeries?: number[],): EChartsOption { return { grid: { height: this.height, @@ -99,17 +102,18 @@ export class LbtcPegsGraphComponent implements OnInit, OnChanges { type: 'line', }, formatter: (params: any) => { - const colorSpan = (color: string) => ``; + const colorSpan = (color: string) => ``; let itemFormatted = '
' + params[0].axisValue + '
'; - params.map((item: any, index: number) => { + for (let index = params.length - 1; index >= 0; index--) { + const item = params[index]; if (index < 26) { itemFormatted += `
${colorSpan(item.color)}
-
-
${formatNumber(item.value, this.locale, '1.2-2')} L-BTC
+
+
${formatNumber(item.value, this.locale, '1.2-2')} ${item.seriesName}
`; } - }); + } return `
${itemFormatted}
`; } }, @@ -138,20 +142,34 @@ export class LbtcPegsGraphComponent implements OnInit, OnChanges { }, series: [ { - data: series, + data: pegSeries, + name: 'L-BTC', + color: '#116761', type: 'line', stack: 'total', - smooth: false, + smooth: true, showSymbol: false, areaStyle: { opacity: 0.2, color: '#116761', }, lineStyle: { - width: 3, + width: 2, color: '#116761', }, }, + { + data: reservesSeries, + name: 'BTC', + color: '#EA983B', + type: 'line', + smooth: true, + showSymbol: false, + lineStyle: { + width: 2, + color: '#EA983B', + }, + }, ], }; } diff --git a/frontend/src/app/components/liquid-master-page/liquid-master-page.component.html b/frontend/src/app/components/liquid-master-page/liquid-master-page.component.html index 49f05c3a2..dd07645bf 100644 --- a/frontend/src/app/components/liquid-master-page/liquid-master-page.component.html +++ b/frontend/src/app/components/liquid-master-page/liquid-master-page.component.html @@ -78,7 +78,7 @@
-
diff --git a/frontend/src/app/components/menu/menu.component.html b/frontend/src/app/components/menu/menu.component.html index 1cc7bdd03..c72016ef5 100644 --- a/frontend/src/app/components/menu/menu.component.html +++ b/frontend/src/app/components/menu/menu.component.html @@ -2,8 +2,19 @@