From 69e6b164b98d4f87ecf10bb7f367027be35a7961 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 20 Jun 2023 16:21:54 -0400 Subject: [PATCH 01/18] Add audit data replication service --- backend/mempool-config.sample.json | 11 ++ .../__fixtures__/mempool-config.template.json | 6 + backend/src/__tests__/config.test.ts | 7 + backend/src/config.ts | 14 ++ backend/src/indexer.ts | 2 + backend/src/mempool.interfaces.ts | 9 ++ backend/src/replication/AuditReplication.ts | 123 ++++++++++++++++++ backend/src/replication/replicator.ts | 70 ++++++++++ production/mempool-config.mainnet.json | 25 ++++ 9 files changed, 267 insertions(+) create mode 100644 backend/src/replication/AuditReplication.ts create mode 100644 backend/src/replication/replicator.ts diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index c0a2d9d62..e3df7d2fe 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -125,5 +125,16 @@ "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1", "BISQ_URL": "https://bisq.markets/api", "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api" + }, + "REPLICATION": { + "ENABLED": false, + "AUDIT": false, + "AUDIT_START_HEIGHT": 774000, + "SERVERS": [ + "list", + "of", + "trusted", + "servers" + ] } } diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 776f01de1..4213f0ffb 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -121,5 +121,11 @@ }, "CLIGHTNING": { "SOCKET": "__CLIGHTNING_SOCKET__" + }, + "REPLICATION": { + "ENABLED": false, + "AUDIT": false, + "AUDIT_START_HEIGHT": 774000, + "SERVERS": [] } } diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index fdd8a02de..dc1beaa46 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -120,6 +120,13 @@ describe('Mempool Backend Config', () => { GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb', GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb' }); + + expect(config.REPLICATION).toStrictEqual({ + ENABLED: false, + AUDIT: false, + AUDIT_START_HEIGHT: 774000, + SERVERS: [] + }); }); }); diff --git a/backend/src/config.ts b/backend/src/config.ts index 40b407a57..09d279537 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -132,6 +132,12 @@ interface IConfig { GEOLITE2_ASN: string; GEOIP2_ISP: string; }, + REPLICATION: { + ENABLED: boolean; + AUDIT: boolean; + AUDIT_START_HEIGHT: number; + SERVERS: string[]; + } } const defaults: IConfig = { @@ -264,6 +270,12 @@ const defaults: IConfig = { 'GEOLITE2_ASN': '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb', 'GEOIP2_ISP': '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb' }, + 'REPLICATION': { + 'ENABLED': false, + 'AUDIT': false, + 'AUDIT_START_HEIGHT': 774000, + 'SERVERS': [], + } }; class Config implements IConfig { @@ -283,6 +295,7 @@ class Config implements IConfig { PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; MAXMIND: IConfig['MAXMIND']; + REPLICATION: IConfig['REPLICATION']; constructor() { const configs = this.merge(configFromFile, defaults); @@ -302,6 +315,7 @@ class Config implements IConfig { this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; this.MAXMIND = configs.MAXMIND; + this.REPLICATION = configs.REPLICATION; } merge = (...objects: object[]): IConfig => { diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 88f44d587..d89a2647f 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -7,6 +7,7 @@ import bitcoinClient from './api/bitcoin/bitcoin-client'; import priceUpdater from './tasks/price-updater'; import PricesRepository from './repositories/PricesRepository'; import config from './config'; +import auditReplicator from './replication/AuditReplication'; export interface CoreIndex { name: string; @@ -136,6 +137,7 @@ class Indexer { await blocks.$generateBlocksSummariesDatabase(); await blocks.$generateCPFPDatabase(); await blocks.$generateAuditStats(); + await auditReplicator.$sync(); } catch (e) { this.indexerRunning = false; logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index a051eea4f..1971234f8 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -236,6 +236,15 @@ export interface BlockSummary { transactions: TransactionStripped[]; } +export interface AuditSummary extends BlockAudit { + timestamp?: number, + size?: number, + weight?: number, + tx_count?: number, + transactions: TransactionStripped[]; + template?: TransactionStripped[]; +} + export interface BlockPrice { height: number; priceId: number; diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts new file mode 100644 index 000000000..b950acb6c --- /dev/null +++ b/backend/src/replication/AuditReplication.ts @@ -0,0 +1,123 @@ +import DB from '../database'; +import logger from '../logger'; +import { AuditSummary } from '../mempool.interfaces'; +import blocksAuditsRepository from '../repositories/BlocksAuditsRepository'; +import blocksSummariesRepository from '../repositories/BlocksSummariesRepository'; +import { $sync } from './replicator'; +import config from '../config'; +import { Common } from '../api/common'; + +const BATCH_SIZE = 16; + +/** + * Syncs missing block template and audit data from trusted servers + */ +class AuditReplication { + inProgress: boolean = false; + skip: Set = new Set(); + + public async $sync(): Promise { + if (!config.REPLICATION.ENABLED || !config.REPLICATION.AUDIT) { + // replication not enabled + return; + } + if (this.inProgress) { + logger.info(`AuditReplication sync already in progress`, 'Replication'); + return; + } + this.inProgress = true; + + const missingAudits = await this.$getMissingAuditBlocks(); + + logger.debug(`Fetching missing audit data for ${missingAudits.length} blocks from trusted servers`, 'Replication'); + + let totalSynced = 0; + let totalMissed = 0; + let loggerTimer = Date.now(); + // process missing audits in batches of + for (let i = 0; i < missingAudits.length; i += BATCH_SIZE) { + const results = await Promise.all(missingAudits.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE).map(hash => this.$syncAudit(hash))); + const synced = results.reduce((total, status) => status ? total + 1 : total, 0); + totalSynced += synced; + totalMissed += (BATCH_SIZE - synced); + if (Date.now() - loggerTimer > 10000) { + loggerTimer = Date.now(); + logger.info(`Found ${totalSynced} / ${totalSynced + totalMissed} of ${missingAudits.length} missing audits`, 'Replication'); + } + await Common.sleep$(1000); + } + + logger.debug(`Fetched ${totalSynced} audits, ${totalMissed} still missing`, 'Replication'); + + this.inProgress = false; + } + + private async $syncAudit(hash: string): Promise { + if (this.skip.has(hash)) { + // we already know none of our trusted servers have this audit + return false; + } + + let success = false; + // start with a random server so load is uniformly spread + const syncResult = await $sync(`/api/v1/block/${hash}/audit-summary`); + if (syncResult) { + if (syncResult.data?.template?.length) { + await this.$saveAuditData(hash, syncResult.data); + success = true; + } + if (!syncResult.data && !syncResult.exists) { + this.skip.add(hash); + } + } + + return success; + } + + private async $getMissingAuditBlocks(): Promise { + try { + const startHeight = config.REPLICATION.AUDIT_START_HEIGHT || 0; + const [rows]: any[] = await DB.query(` + SELECT auditable.hash, auditable.height + FROM ( + SELECT hash, height + FROM blocks + WHERE height >= ? + ) AS auditable + LEFT JOIN blocks_audits ON auditable.hash = blocks_audits.hash + WHERE blocks_audits.hash IS NULL + ORDER BY auditable.height DESC + `, [startHeight]); + return rows.map(row => row.hash); + } catch (e: any) { + logger.err(`Cannot fetch missing audit blocks from db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + private async $saveAuditData(blockHash: string, auditSummary: AuditSummary): Promise { + // save audit & template to DB + await blocksSummariesRepository.$saveTemplate({ + height: auditSummary.height, + template: { + id: blockHash, + transactions: auditSummary.template || [] + } + }); + await blocksAuditsRepository.$saveAudit({ + hash: blockHash, + height: auditSummary.height, + time: auditSummary.timestamp || auditSummary.time, + missingTxs: auditSummary.missingTxs || [], + addedTxs: auditSummary.addedTxs || [], + freshTxs: auditSummary.freshTxs || [], + sigopTxs: auditSummary.sigopTxs || [], + matchRate: auditSummary.matchRate, + expectedFees: auditSummary.expectedFees, + expectedWeight: auditSummary.expectedWeight, + }); + } +} + +export default new AuditReplication(); + diff --git a/backend/src/replication/replicator.ts b/backend/src/replication/replicator.ts new file mode 100644 index 000000000..60dfa8a2d --- /dev/null +++ b/backend/src/replication/replicator.ts @@ -0,0 +1,70 @@ +import config from '../config'; +import backendInfo from '../api/backend-info'; +import axios, { AxiosResponse } from 'axios'; +import { SocksProxyAgent } from 'socks-proxy-agent'; +import * as https from 'https'; + +export async function $sync(path): Promise<{ data?: any, exists: boolean }> { + // start with a random server so load is uniformly spread + let allMissing = true; + const offset = Math.floor(Math.random() * config.REPLICATION.SERVERS.length); + for (let i = 0; i < config.REPLICATION.SERVERS.length; i++) { + const server = config.REPLICATION.SERVERS[(i + offset) % config.REPLICATION.SERVERS.length]; + // don't query ourself + if (server === backendInfo.getBackendInfo().hostname) { + continue; + } + + try { + const result = await query(`https://${server}${path}`); + if (result) { + return { data: result, exists: true }; + } + } catch (e: any) { + if (e?.response?.status === 404) { + // this server is also missing this data + } else { + // something else went wrong + allMissing = false; + } + } + } + + return { exists: !allMissing }; +} + +export async function query(path): Promise { + type axiosOptions = { + headers: { + 'User-Agent': string + }; + timeout: number; + httpsAgent?: https.Agent; + }; + const axiosOptions: axiosOptions = { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, + timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 + }; + + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT, + username: config.SOCKS5PROXY.USERNAME || 'circuit0', + password: config.SOCKS5PROXY.PASSWORD, + }; + + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + + const data: AxiosResponse = await axios.get(path, axiosOptions); + if (data.statusText === 'error' || !data.data) { + throw new Error(`${data.status}`); + } + return data.data; +} \ No newline at end of file diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index a76053913..5e25bcb76 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -48,5 +48,30 @@ "STATISTICS": { "ENABLED": true, "TX_PER_SECOND_SAMPLE_PERIOD": 150 + }, + "REPLICATION": { + "ENABLED": true, + "AUDIT": true, + "AUDIT_START_HEIGHT": 774000, + "SERVERS": [ + "node201.fmt.mempool.space", + "node202.fmt.mempool.space", + "node203.fmt.mempool.space", + "node204.fmt.mempool.space", + "node205.fmt.mempool.space", + "node206.fmt.mempool.space", + "node201.fra.mempool.space", + "node202.fra.mempool.space", + "node203.fra.mempool.space", + "node204.fra.mempool.space", + "node205.fra.mempool.space", + "node206.fra.mempool.space", + "node201.tk7.mempool.space", + "node202.tk7.mempool.space", + "node203.tk7.mempool.space", + "node204.tk7.mempool.space", + "node205.tk7.mempool.space", + "node206.tk7.mempool.space" + ] } } From 736b997104c592673fd3e558f94d3eac1be2071d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 20 Jun 2023 17:03:24 -0400 Subject: [PATCH 02/18] Add missing audit data to cached blocks --- backend/src/replication/AuditReplication.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index b950acb6c..c762df201 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -6,6 +6,7 @@ import blocksSummariesRepository from '../repositories/BlocksSummariesRepository import { $sync } from './replicator'; import config from '../config'; import { Common } from '../api/common'; +import blocks from '../api/blocks'; const BATCH_SIZE = 16; @@ -116,6 +117,13 @@ class AuditReplication { expectedFees: auditSummary.expectedFees, expectedWeight: auditSummary.expectedWeight, }); + // add missing data to cached blocks + const cachedBlock = blocks.getBlocks().find(block => block.id === blockHash); + if (cachedBlock) { + cachedBlock.extras.matchRate = auditSummary.matchRate; + cachedBlock.extras.expectedFees = auditSummary.expectedFees || null; + cachedBlock.extras.expectedWeight = auditSummary.expectedWeight || null; + } } } From 7f6d17fc0ece0bfc57945fafca338a96ce0e64c1 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 20 Jun 2023 17:07:14 -0400 Subject: [PATCH 03/18] Fix audit sync progress logging --- backend/src/replication/AuditReplication.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index c762df201..89c514347 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -37,10 +37,11 @@ class AuditReplication { let loggerTimer = Date.now(); // process missing audits in batches of for (let i = 0; i < missingAudits.length; i += BATCH_SIZE) { - const results = await Promise.all(missingAudits.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE).map(hash => this.$syncAudit(hash))); + const slice = missingAudits.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE); + const results = await Promise.all(slice.map(hash => this.$syncAudit(hash))); const synced = results.reduce((total, status) => status ? total + 1 : total, 0); totalSynced += synced; - totalMissed += (BATCH_SIZE - synced); + totalMissed += (slice.length - synced); if (Date.now() - loggerTimer > 10000) { loggerTimer = Date.now(); logger.info(`Found ${totalSynced} / ${totalSynced + totalMissed} of ${missingAudits.length} missing audits`, 'Replication'); From bccc6b3680e4bd6aeeb25bc229983de1f21b8c91 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 21 Jun 2023 09:19:59 -0400 Subject: [PATCH 04/18] Add missing replication docker config --- docker/backend/mempool-config.json | 6 ++++++ docker/backend/start.sh | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index d070d8010..2ff76d5dd 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -127,5 +127,11 @@ "GEOLITE2_CITY": "__MAXMIND_GEOLITE2_CITY__", "GEOLITE2_ASN": "__MAXMIND_GEOLITE2_ASN__", "GEOIP2_ISP": "__MAXMIND_GEOIP2_ISP__" + }, + "REPLICATION": { + "ENABLED": __REPLICATION_ENABLED__, + "AUDIT": __REPLICATION_AUDIT__, + "AUDIT_START_HEIGHT": __REPLICATION_AUDIT_START_HEIGHT__, + "SERVERS": __REPLICATION_SERVERS__ } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 7241444fb..c34d804b4 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -130,6 +130,12 @@ __MAXMIND_GEOLITE2_CITY__=${MAXMIND_GEOLITE2_CITY:="/backend/GeoIP/GeoLite2-City __MAXMIND_GEOLITE2_ASN__=${MAXMIND_GEOLITE2_ASN:="/backend/GeoIP/GeoLite2-ASN.mmdb"} __MAXMIND_GEOIP2_ISP__=${MAXMIND_GEOIP2_ISP:=""} +# REPLICATION +__REPLICATION_ENABLED__=${REPLICATION_ENABLED:=true} +__REPLICATION_AUDIT__=${REPLICATION_AUDIT:=true} +__REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000} +__REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]} + mkdir -p "${__MEMPOOL_CACHE_DIR__}" @@ -250,5 +256,10 @@ sed -i "s!__MAXMIND_GEOLITE2_CITY__!${__MAXMIND_GEOLITE2_CITY__}!g" mempool-conf sed -i "s!__MAXMIND_GEOLITE2_ASN__!${__MAXMIND_GEOLITE2_ASN__}!g" mempool-config.json sed -i "s!__MAXMIND_GEOIP2_ISP__!${__MAXMIND_GEOIP2_ISP__}!g" mempool-config.json +# REPLICATION +sed -i "s!__REPLICATION_ENABLED__!${__REPLICATION_ENABLED__}!g" mempool-config.json +sed -i "s!__REPLICATION_AUDIT__!${__REPLICATION_AUDIT__}!g" mempool-config.json +sed -i "s!__REPLICATION_AUDIT_START_HEIGHT__!${__REPLICATION_AUDIT_START_HEIGHT__}!g" mempool-config.json +sed -i "s!__REPLICATION_SERVERS__!${__REPLICATION_SERVERS__}!g" mempool-config.json node /backend/package/index.js From e59a9d38ff238850fc9e727102a56ff0f5c0e05e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 14 Jul 2023 16:47:58 +0900 Subject: [PATCH 05/18] fix audit replication merge conflicts --- backend/src/replication/AuditReplication.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index 89c514347..2043532db 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -37,7 +37,7 @@ class AuditReplication { let loggerTimer = Date.now(); // process missing audits in batches of for (let i = 0; i < missingAudits.length; i += BATCH_SIZE) { - const slice = missingAudits.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE); + const slice = missingAudits.slice(i, i + BATCH_SIZE); const results = await Promise.all(slice.map(hash => this.$syncAudit(hash))); const synced = results.reduce((total, status) => status ? total + 1 : total, 0); totalSynced += synced; @@ -114,6 +114,7 @@ class AuditReplication { addedTxs: auditSummary.addedTxs || [], freshTxs: auditSummary.freshTxs || [], sigopTxs: auditSummary.sigopTxs || [], + fullrbfTxs: auditSummary.fullrbfTxs || [], matchRate: auditSummary.matchRate, expectedFees: auditSummary.expectedFees, expectedWeight: auditSummary.expectedWeight, From 1abd2a23cce04a8913791eb88a0818814f35d106 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 14 Jul 2023 16:48:11 +0900 Subject: [PATCH 06/18] Add audit replication success logging --- backend/src/replication/AuditReplication.ts | 1 + backend/src/replication/replicator.ts | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index 2043532db..26bf6dad7 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -66,6 +66,7 @@ class AuditReplication { if (syncResult) { if (syncResult.data?.template?.length) { await this.$saveAuditData(hash, syncResult.data); + logger.info(`Imported audit data from ${syncResult.server} for block ${syncResult.data.height} (${hash})`); success = true; } if (!syncResult.data && !syncResult.exists) { diff --git a/backend/src/replication/replicator.ts b/backend/src/replication/replicator.ts index 60dfa8a2d..ac204efcc 100644 --- a/backend/src/replication/replicator.ts +++ b/backend/src/replication/replicator.ts @@ -4,7 +4,7 @@ import axios, { AxiosResponse } from 'axios'; import { SocksProxyAgent } from 'socks-proxy-agent'; import * as https from 'https'; -export async function $sync(path): Promise<{ data?: any, exists: boolean }> { +export async function $sync(path): Promise<{ data?: any, exists: boolean, server?: string }> { // start with a random server so load is uniformly spread let allMissing = true; const offset = Math.floor(Math.random() * config.REPLICATION.SERVERS.length); @@ -18,7 +18,7 @@ export async function $sync(path): Promise<{ data?: any, exists: boolean }> { try { const result = await query(`https://${server}${path}`); if (result) { - return { data: result, exists: true }; + return { data: result, exists: true, server }; } } catch (e: any) { if (e?.response?.status === 404) { From 67a998c69f165905c2e720566e78f79a2dd02c56 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 17 Jun 2023 21:04:23 +0200 Subject: [PATCH 07/18] Working fiat/btc calculator --- frontend/src/app/app-routing.module.ts | 5 ++ .../calculator/calculator.component.html | 44 ++++++++++ .../calculator/calculator.component.scss | 3 + .../calculator/calculator.component.ts | 85 +++++++++++++++++++ .../svg-images/svg-images.component.html | 3 + frontend/src/app/shared/shared.module.ts | 4 +- 6 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 frontend/src/app/components/calculator/calculator.component.html create mode 100644 frontend/src/app/components/calculator/calculator.component.scss create mode 100644 frontend/src/app/components/calculator/calculator.component.ts diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index 7a9b53ed0..c7982b75f 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -22,6 +22,7 @@ import { AssetsFeaturedComponent } from './components/assets/assets-featured/ass import { AssetsComponent } from './components/assets/assets.component'; import { AssetComponent } from './components/asset/asset.component'; import { AssetsNavComponent } from './components/assets/assets-nav/assets-nav.component'; +import { CalculatorComponent } from './components/calculator/calculator.component'; const browserWindow = window || {}; // @ts-ignore @@ -278,6 +279,10 @@ let routes: Routes = [ path: 'rbf', component: RbfList, }, + { + path: 'calculator', + component: CalculatorComponent + }, { path: 'terms-of-service', component: TermsOfServiceComponent diff --git a/frontend/src/app/components/calculator/calculator.component.html b/frontend/src/app/components/calculator/calculator.component.html new file mode 100644 index 000000000..62026566f --- /dev/null +++ b/frontend/src/app/components/calculator/calculator.component.html @@ -0,0 +1,44 @@ +
+
+

Calculator

+
+ +
+
+ +
+
+
+ {{ currency$ | async }} +
+ + +
+ +
+
+ +
+ + +
+ +
+
+ +
+ + +
+
+ +
+
+ + +
+ Waiting for price feed... +
+ + +
\ No newline at end of file diff --git a/frontend/src/app/components/calculator/calculator.component.scss b/frontend/src/app/components/calculator/calculator.component.scss new file mode 100644 index 000000000..bc3ca2665 --- /dev/null +++ b/frontend/src/app/components/calculator/calculator.component.scss @@ -0,0 +1,3 @@ +.input-group-text { + width: 75px; +} diff --git a/frontend/src/app/components/calculator/calculator.component.ts b/frontend/src/app/components/calculator/calculator.component.ts new file mode 100644 index 000000000..f857bbd8c --- /dev/null +++ b/frontend/src/app/components/calculator/calculator.component.ts @@ -0,0 +1,85 @@ +import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { combineLatest, Observable } from 'rxjs'; +import { map, switchMap } from 'rxjs/operators'; +import { Price, PriceService } from '../../services/price.service'; +import { StateService } from '../../services/state.service'; +import { WebsocketService } from '../../services/websocket.service'; + +@Component({ + selector: 'app-calculator', + templateUrl: './calculator.component.html', + styleUrls: ['./calculator.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class CalculatorComponent implements OnInit { + satoshis = 10000; + form: FormGroup; + + currency: string; + currency$ = this.stateService.fiatCurrency$; + mainSubscription$: Observable; + price$: Observable; + + constructor( + private priceService: PriceService, + private stateService: StateService, + private formBuilder: FormBuilder, + private websocketService: WebsocketService, + ) { } + + ngOnInit(): void { + this.form = this.formBuilder.group({ + fiat: [0], + bitcoin: [0], + satoshis: [0], + }); + + this.price$ = this.currency$.pipe( + switchMap((currency) => { + this.currency = currency; + return this.stateService.conversions$.asObservable(); + }), + map((conversions) => { + return conversions[this.currency]; + }) + ); + + combineLatest([ + this.price$, + this.form.get('fiat').valueChanges + ]).subscribe(([price, value]) => { + value = parseFloat(value.replace(',', '.')); + value = value || 0; + const rate = (value / price).toFixed(8); + const satsRate = Math.round(value / price * 100_000_000); + this.form.get('bitcoin').setValue(rate, { emitEvent: false }); + this.form.get('satoshis').setValue(satsRate, { emitEvent: false } ); + }); + + combineLatest([ + this.price$, + this.form.get('bitcoin').valueChanges + ]).subscribe(([price, value]) => { + value = parseFloat(value.replace(',', '.')); + value = value || 0; + const rate = parseFloat((value * price).toFixed(8)); + this.form.get('fiat').setValue(rate, { emitEvent: false } ); + this.form.get('satoshis').setValue(Math.round(value * 100_000_000), { emitEvent: false } ); + }); + + combineLatest([ + this.price$, + this.form.get('satoshis').valueChanges + ]).subscribe(([price, value]) => { + value = parseFloat(value.replace(',', '.')); + value = value || 0; + const rate = parseFloat((value / 100_000_000 * price).toFixed(8)); + const bitcoinRate = (value / 100_000_000).toFixed(8); + this.form.get('fiat').setValue(rate, { emitEvent: false } ); + this.form.get('bitcoin').setValue(bitcoinRate, { emitEvent: false }); + }); + + } + +} diff --git a/frontend/src/app/components/svg-images/svg-images.component.html b/frontend/src/app/components/svg-images/svg-images.component.html index c4d5296bd..1c3a8bc2d 100644 --- a/frontend/src/app/components/svg-images/svg-images.component.html +++ b/frontend/src/app/components/svg-images/svg-images.component.html @@ -74,6 +74,9 @@ + + + diff --git a/frontend/src/app/shared/shared.module.ts b/frontend/src/app/shared/shared.module.ts index 9cf780116..4a114faa7 100644 --- a/frontend/src/app/shared/shared.module.ts +++ b/frontend/src/app/shared/shared.module.ts @@ -97,6 +97,7 @@ import { MempoolBlockOverviewComponent } from '../components/mempool-block-overv import { ClockchainComponent } from '../components/clockchain/clockchain.component'; import { ClockFaceComponent } from '../components/clock-face/clock-face.component'; import { ClockComponent } from '../components/clock/clock.component'; +import { CalculatorComponent } from '../components/calculator/calculator.component'; import { OnlyVsizeDirective, OnlyWeightDirective } from './components/weight-directives/weight-directives'; @@ -185,12 +186,11 @@ import { OnlyVsizeDirective, OnlyWeightDirective } from './components/weight-dir GeolocationComponent, TestnetAlertComponent, GlobalFooterComponent, - + CalculatorComponent, MempoolBlockOverviewComponent, ClockchainComponent, ClockComponent, ClockFaceComponent, - OnlyVsizeDirective, OnlyWeightDirective ], From 120c27d120ba28611e9b12e008a0054b9ee4c4eb Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 18 Jun 2023 00:16:47 +0200 Subject: [PATCH 08/18] Calculator visual results --- frontend/src/app/app-routing.module.ts | 2 +- .../calculator/calculator.component.html | 41 +++++++++++++---- .../calculator/calculator.component.scss | 18 ++++++++ .../calculator/calculator.component.ts | 44 +++++++++++++------ .../app/shared/pipes/bitcoinsatoshis.pipe.ts | 28 ++++++++++++ frontend/src/app/shared/shared.module.ts | 5 ++- 6 files changed, 115 insertions(+), 23 deletions(-) create mode 100644 frontend/src/app/shared/pipes/bitcoinsatoshis.pipe.ts diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index c7982b75f..79a8e1c02 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -280,7 +280,7 @@ let routes: Routes = [ component: RbfList, }, { - path: 'calculator', + path: 'tools/calculator', component: CalculatorComponent }, { diff --git a/frontend/src/app/components/calculator/calculator.component.html b/frontend/src/app/components/calculator/calculator.component.html index 62026566f..da8e77e79 100644 --- a/frontend/src/app/components/calculator/calculator.component.html +++ b/frontend/src/app/components/calculator/calculator.component.html @@ -3,37 +3,62 @@

Calculator

-
-
+ + +
{{ currency$ | async }}
- +
- +
- +
- +
- +
-
+ +
+ +
+
+ + + +
+
+ +
+
+ +
+
+ +
+
+ Fiat price last updated +
+
+ + +
diff --git a/frontend/src/app/components/calculator/calculator.component.scss b/frontend/src/app/components/calculator/calculator.component.scss index bc3ca2665..649af4934 100644 --- a/frontend/src/app/components/calculator/calculator.component.scss +++ b/frontend/src/app/components/calculator/calculator.component.scss @@ -1,3 +1,21 @@ .input-group-text { width: 75px; } + +.bitcoin-satoshis-text { + font-size: 40px; +} + +.fiat-text { + font-size: 24px; +} + +.symbol { + font-style: italic; +} + +@media (max-width: 767.98px) { + .bitcoin-satoshis-text { + font-size: 30px; + } +} \ No newline at end of file diff --git a/frontend/src/app/components/calculator/calculator.component.ts b/frontend/src/app/components/calculator/calculator.component.ts index f857bbd8c..838afbbd4 100644 --- a/frontend/src/app/components/calculator/calculator.component.ts +++ b/frontend/src/app/components/calculator/calculator.component.ts @@ -2,7 +2,6 @@ import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; import { FormBuilder, FormGroup } from '@angular/forms'; import { combineLatest, Observable } from 'rxjs'; import { map, switchMap } from 'rxjs/operators'; -import { Price, PriceService } from '../../services/price.service'; import { StateService } from '../../services/state.service'; import { WebsocketService } from '../../services/websocket.service'; @@ -16,13 +15,11 @@ export class CalculatorComponent implements OnInit { satoshis = 10000; form: FormGroup; - currency: string; currency$ = this.stateService.fiatCurrency$; - mainSubscription$: Observable; price$: Observable; + lastFiatPrice$: Observable; constructor( - private priceService: PriceService, private stateService: StateService, private formBuilder: FormBuilder, private websocketService: WebsocketService, @@ -35,13 +32,19 @@ export class CalculatorComponent implements OnInit { satoshis: [0], }); + this.lastFiatPrice$ = this.stateService.conversions$.asObservable() + .pipe( + map((conversions) => conversions.time) + ); + + let currency; this.price$ = this.currency$.pipe( - switchMap((currency) => { - this.currency = currency; + switchMap((result) => { + currency = result; return this.stateService.conversions$.asObservable(); }), map((conversions) => { - return conversions[this.currency]; + return conversions[currency]; }) ); @@ -49,8 +52,6 @@ export class CalculatorComponent implements OnInit { this.price$, this.form.get('fiat').valueChanges ]).subscribe(([price, value]) => { - value = parseFloat(value.replace(',', '.')); - value = value || 0; const rate = (value / price).toFixed(8); const satsRate = Math.round(value / price * 100_000_000); this.form.get('bitcoin').setValue(rate, { emitEvent: false }); @@ -61,8 +62,6 @@ export class CalculatorComponent implements OnInit { this.price$, this.form.get('bitcoin').valueChanges ]).subscribe(([price, value]) => { - value = parseFloat(value.replace(',', '.')); - value = value || 0; const rate = parseFloat((value * price).toFixed(8)); this.form.get('fiat').setValue(rate, { emitEvent: false } ); this.form.get('satoshis').setValue(Math.round(value * 100_000_000), { emitEvent: false } ); @@ -72,8 +71,6 @@ export class CalculatorComponent implements OnInit { this.price$, this.form.get('satoshis').valueChanges ]).subscribe(([price, value]) => { - value = parseFloat(value.replace(',', '.')); - value = value || 0; const rate = parseFloat((value / 100_000_000 * price).toFixed(8)); const bitcoinRate = (value / 100_000_000).toFixed(8); this.form.get('fiat').setValue(rate, { emitEvent: false } ); @@ -82,4 +79,25 @@ export class CalculatorComponent implements OnInit { } + transformInput(name: string): void { + const formControl = this.form.get(name); + if (!formControl.value) { + return formControl.setValue('', {emitEvent: false}); + } + let value = formControl.value.replace(',', '.').replace(/[^0-9.]/g, ''); + if (value === '.') { + value = '0'; + } + const sanitizedValue = this.removeExtraDots(value); + formControl.setValue(sanitizedValue, {emitEvent: true}); + } + + removeExtraDots(str: string): string { + const [beforeDot, afterDot] = str.split('.', 2); + if (afterDot === undefined) { + return str; + } + const afterDotReplaced = afterDot.replace(/\./g, ''); + return `${beforeDot}.${afterDotReplaced}`; + } } diff --git a/frontend/src/app/shared/pipes/bitcoinsatoshis.pipe.ts b/frontend/src/app/shared/pipes/bitcoinsatoshis.pipe.ts new file mode 100644 index 000000000..7065b5138 --- /dev/null +++ b/frontend/src/app/shared/pipes/bitcoinsatoshis.pipe.ts @@ -0,0 +1,28 @@ +import { Pipe, PipeTransform } from '@angular/core'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Pipe({ + name: 'bitcoinsatoshis' +}) +export class BitcoinsatoshisPipe implements PipeTransform { + + constructor(private sanitizer: DomSanitizer) { } + + transform(value: string): SafeHtml { + const newValue = this.insertSpaces(parseFloat(value || '0').toFixed(8)); + const position = (newValue || '0').search(/[1-9]/); + + const firstPart = newValue.slice(0, position); + const secondPart = newValue.slice(position); + + return this.sanitizer.bypassSecurityTrustHtml( + `${firstPart}${secondPart}` + ); + } + + insertSpaces(str: string): string { + const length = str.length; + return str.slice(0, length - 6) + ' ' + str.slice(length - 6, length - 3) + ' ' + str.slice(length - 3); + } + +} diff --git a/frontend/src/app/shared/shared.module.ts b/frontend/src/app/shared/shared.module.ts index 4a114faa7..c06b1dc8f 100644 --- a/frontend/src/app/shared/shared.module.ts +++ b/frontend/src/app/shared/shared.module.ts @@ -98,6 +98,7 @@ import { ClockchainComponent } from '../components/clockchain/clockchain.compone import { ClockFaceComponent } from '../components/clock-face/clock-face.component'; import { ClockComponent } from '../components/clock/clock.component'; import { CalculatorComponent } from '../components/calculator/calculator.component'; +import { BitcoinsatoshisPipe } from '../shared/pipes/bitcoinsatoshis.pipe'; import { OnlyVsizeDirective, OnlyWeightDirective } from './components/weight-directives/weight-directives'; @@ -190,9 +191,11 @@ import { OnlyVsizeDirective, OnlyWeightDirective } from './components/weight-dir MempoolBlockOverviewComponent, ClockchainComponent, ClockComponent, + CalculatorComponent, ClockFaceComponent, OnlyVsizeDirective, - OnlyWeightDirective + OnlyWeightDirective, + BitcoinsatoshisPipe ], imports: [ CommonModule, From 98be07f5efd4ff622c1b74647cde7ef324e6e242 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 18 Jun 2023 02:09:06 +0200 Subject: [PATCH 09/18] Removing logos --- .../app/components/calculator/calculator.component.html | 8 ++++---- .../app/components/calculator/calculator.component.scss | 4 ++++ .../app/components/svg-images/svg-images.component.html | 3 --- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/frontend/src/app/components/calculator/calculator.component.html b/frontend/src/app/components/calculator/calculator.component.html index da8e77e79..df2146760 100644 --- a/frontend/src/app/components/calculator/calculator.component.html +++ b/frontend/src/app/components/calculator/calculator.component.html @@ -18,7 +18,7 @@
- + BTC
@@ -26,7 +26,7 @@
- + sats
@@ -39,9 +39,9 @@
- + ₿ - + sats
diff --git a/frontend/src/app/components/calculator/calculator.component.scss b/frontend/src/app/components/calculator/calculator.component.scss index 649af4934..c9d608455 100644 --- a/frontend/src/app/components/calculator/calculator.component.scss +++ b/frontend/src/app/components/calculator/calculator.component.scss @@ -18,4 +18,8 @@ .bitcoin-satoshis-text { font-size: 30px; } +} + +.sats { + font-size: 25px; } \ No newline at end of file diff --git a/frontend/src/app/components/svg-images/svg-images.component.html b/frontend/src/app/components/svg-images/svg-images.component.html index 1c3a8bc2d..c4d5296bd 100644 --- a/frontend/src/app/components/svg-images/svg-images.component.html +++ b/frontend/src/app/components/svg-images/svg-images.component.html @@ -74,9 +74,6 @@ - - - From 23dffb4ca27d595809e6003b88545be0a87a16cb Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 18 Jun 2023 18:04:32 +0200 Subject: [PATCH 10/18] Slight margin fix --- .../src/app/components/calculator/calculator.component.scss | 3 ++- frontend/src/app/shared/shared.module.ts | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/components/calculator/calculator.component.scss b/frontend/src/app/components/calculator/calculator.component.scss index c9d608455..5c0cbb5b1 100644 --- a/frontend/src/app/components/calculator/calculator.component.scss +++ b/frontend/src/app/components/calculator/calculator.component.scss @@ -21,5 +21,6 @@ } .sats { - font-size: 25px; + font-size: 20px; + margin-left: 5px; } \ No newline at end of file diff --git a/frontend/src/app/shared/shared.module.ts b/frontend/src/app/shared/shared.module.ts index c06b1dc8f..d56986107 100644 --- a/frontend/src/app/shared/shared.module.ts +++ b/frontend/src/app/shared/shared.module.ts @@ -188,14 +188,13 @@ import { OnlyVsizeDirective, OnlyWeightDirective } from './components/weight-dir TestnetAlertComponent, GlobalFooterComponent, CalculatorComponent, + BitcoinsatoshisPipe, MempoolBlockOverviewComponent, ClockchainComponent, ClockComponent, - CalculatorComponent, ClockFaceComponent, OnlyVsizeDirective, - OnlyWeightDirective, - BitcoinsatoshisPipe + OnlyWeightDirective ], imports: [ CommonModule, From 9ffd4cc38d804cb498c422c9c481760e99fec4e3 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 15 Jul 2023 12:18:55 +0900 Subject: [PATCH 11/18] Calculator mobile margin --- .../src/app/components/calculator/calculator.component.scss | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/calculator/calculator.component.scss b/frontend/src/app/components/calculator/calculator.component.scss index 5c0cbb5b1..81f74f9ee 100644 --- a/frontend/src/app/components/calculator/calculator.component.scss +++ b/frontend/src/app/components/calculator/calculator.component.scss @@ -23,4 +23,8 @@ .sats { font-size: 20px; margin-left: 5px; -} \ No newline at end of file +} + +.row { + margin: auto; +} From 992196c91f7600578c079eefeaa77eead08e2a51 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 15 Jul 2023 15:09:41 +0900 Subject: [PATCH 12/18] Calculator validation improvements --- .../calculator/calculator.component.ts | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/calculator/calculator.component.ts b/frontend/src/app/components/calculator/calculator.component.ts index 838afbbd4..d99302f40 100644 --- a/frontend/src/app/components/calculator/calculator.component.ts +++ b/frontend/src/app/components/calculator/calculator.component.ts @@ -54,6 +54,9 @@ export class CalculatorComponent implements OnInit { ]).subscribe(([price, value]) => { const rate = (value / price).toFixed(8); const satsRate = Math.round(value / price * 100_000_000); + if (isNaN(value)) { + return; + } this.form.get('bitcoin').setValue(rate, { emitEvent: false }); this.form.get('satoshis').setValue(satsRate, { emitEvent: false } ); }); @@ -63,6 +66,9 @@ export class CalculatorComponent implements OnInit { this.form.get('bitcoin').valueChanges ]).subscribe(([price, value]) => { const rate = parseFloat((value * price).toFixed(8)); + if (isNaN(value)) { + return; + } this.form.get('fiat').setValue(rate, { emitEvent: false } ); this.form.get('satoshis').setValue(Math.round(value * 100_000_000), { emitEvent: false } ); }); @@ -73,6 +79,9 @@ export class CalculatorComponent implements OnInit { ]).subscribe(([price, value]) => { const rate = parseFloat((value / 100_000_000 * price).toFixed(8)); const bitcoinRate = (value / 100_000_000).toFixed(8); + if (isNaN(value)) { + return; + } this.form.get('fiat').setValue(rate, { emitEvent: false } ); this.form.get('bitcoin').setValue(bitcoinRate, { emitEvent: false }); }); @@ -88,7 +97,16 @@ export class CalculatorComponent implements OnInit { if (value === '.') { value = '0'; } - const sanitizedValue = this.removeExtraDots(value); + let sanitizedValue = this.removeExtraDots(value); + if (name === 'bitcoin' && this.countDecimals(sanitizedValue) > 8) { + sanitizedValue = this.toFixedWithoutRounding(sanitizedValue, 8); + } + if (sanitizedValue === '') { + sanitizedValue = '0'; + } + if (name === 'satoshis') { + sanitizedValue = parseFloat(sanitizedValue).toFixed(0); + } formControl.setValue(sanitizedValue, {emitEvent: true}); } @@ -100,4 +118,16 @@ export class CalculatorComponent implements OnInit { const afterDotReplaced = afterDot.replace(/\./g, ''); return `${beforeDot}.${afterDotReplaced}`; } + + countDecimals(numberString: string): number { + const decimalPos = numberString.indexOf('.'); + if (decimalPos === -1) return 0; + return numberString.length - decimalPos - 1; + } + + toFixedWithoutRounding(numStr: string, fixed: number): string { + const re = new RegExp(`^-?\\d+(?:.\\d{0,${(fixed || -1)}})?`); + const result = numStr.match(re); + return result ? result[0] : numStr; + } } From 73d9b4ef2873bda5a82316c734517407bb3373b0 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 15 Jul 2023 17:29:29 +0900 Subject: [PATCH 13/18] [price updater] update latestPrices timestamp before pushing to websocket --- backend/src/tasks/price-updater.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 3b9dad30e..fafe2b913 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -153,6 +153,7 @@ class PriceUpdater { try { const p = 60 * 60 * 1000; // milliseconds in an hour const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042 + this.latestPrices.time = nowRounded.getTime() / 1000; await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices); } catch (e) { this.lastRun = previousRun + 5 * 60; From b39f01471a492f8dcee347299db152fb09c32009 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 15 Jul 2023 17:47:36 +0900 Subject: [PATCH 14/18] Select all input box text on click --- .../src/app/components/calculator/calculator.component.html | 6 +++--- .../src/app/components/calculator/calculator.component.ts | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/calculator/calculator.component.html b/frontend/src/app/components/calculator/calculator.component.html index df2146760..bdbfdd0cd 100644 --- a/frontend/src/app/components/calculator/calculator.component.html +++ b/frontend/src/app/components/calculator/calculator.component.html @@ -12,7 +12,7 @@
{{ currency$ | async }}
- +
@@ -20,7 +20,7 @@
BTC
- +
@@ -28,7 +28,7 @@
sats
- +
diff --git a/frontend/src/app/components/calculator/calculator.component.ts b/frontend/src/app/components/calculator/calculator.component.ts index d99302f40..a6f10c049 100644 --- a/frontend/src/app/components/calculator/calculator.component.ts +++ b/frontend/src/app/components/calculator/calculator.component.ts @@ -130,4 +130,8 @@ export class CalculatorComponent implements OnInit { const result = numStr.match(re); return result ? result[0] : numStr; } + + selectAll(event): void { + event.target.select(); + } } From b33ea4679ddf0e0b7c9df69908296afd17cd0039 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 16 Jul 2023 13:49:33 +0900 Subject: [PATCH 15/18] Add "recently cpfpd" exception to audits --- backend/src/api/audit.ts | 11 +++++++---- backend/src/api/mempool-blocks.ts | 1 + backend/src/mempool.interfaces.ts | 1 + .../app/components/block-overview-graph/tx-view.ts | 3 ++- .../block-overview-tooltip.component.html | 1 + frontend/src/app/components/block/block.component.ts | 6 +++++- frontend/src/app/interfaces/node-api.interface.ts | 3 ++- frontend/src/app/interfaces/websocket.interface.ts | 2 +- 8 files changed, 20 insertions(+), 8 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index e79196a7a..f7aecfca8 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -1,12 +1,12 @@ import config from '../config'; import logger from '../logger'; -import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; +import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended }) + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 }; @@ -14,7 +14,7 @@ class Audit { const matches: string[] = []; // present in both mined block and template const added: string[] = []; // present in mined block, not in template - const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN + const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN const fullrbf: string[] = []; // either missing or present, and part of a fullrbf replacement const isCensored = {}; // missing, without excuse const isDisplaced = {}; @@ -36,10 +36,13 @@ class Audit { // look for transactions that were expected in the template, but missing from the mined block for (const txid of projectedBlocks[0].transactionIds) { if (!inBlock[txid]) { - // tx is recent, may have reached the miner too late for inclusion if (rbfCache.isFullRbf(txid)) { fullrbf.push(txid); } else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) { + // tx is recent, may have reached the miner too late for inclusion + fresh.push(txid); + } else if (mempool[txid]?.lastBoosted != null && (now - (mempool[txid]?.lastBoosted || 0)) <= PROPAGATION_MARGIN) { + // tx was recently cpfp'd, miner may not have the latest effective rate fresh.push(txid); } else { isCensored[txid] = true; diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index d5538854a..08508310d 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -457,6 +457,7 @@ class MempoolBlocks { }; if (matched) { descendants.push(relative); + mempoolTx.lastBoosted = Math.max(mempoolTx.lastBoosted || 0, mempool[txid].firstSeen || 0); } else { ancestors.push(relative); } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index a051eea4f..3dad451ac 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -100,6 +100,7 @@ export interface MempoolTransactionExtended extends TransactionExtended { adjustedVsize: number; adjustedFeePerVsize: number; inputs?: number[]; + lastBoosted?: number; } export interface AuditTransaction { diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index 77f5a182a..452bb38f5 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -38,7 +38,7 @@ export default class TxView implements TransactionStripped { value: number; feerate: number; rate?: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf'; context?: 'projected' | 'actual'; scene?: BlockScene; @@ -210,6 +210,7 @@ export default class TxView implements TransactionStripped { case 'fullrbf': return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1]; case 'fresh': + case 'freshcpfp': return auditColors.missing; case 'added': return auditColors.added; diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index 5ebd8fceb..59450326b 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -50,6 +50,7 @@ Marginal fee rate High sigop count Recently broadcasted + Recently CPFP'd Added Marginal fee rate Full RBF diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 0d733ff6b..4be6e3aff 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -370,7 +370,11 @@ export class BlockComponent implements OnInit, OnDestroy { tx.status = 'found'; } else { if (isFresh[tx.txid]) { - tx.status = 'fresh'; + if (tx.rate - (tx.fee / tx.vsize) >= 0.1) { + tx.status = 'freshcpfp'; + } else { + tx.status = 'fresh'; + } } else if (isSigop[tx.txid]) { tx.status = 'sigop'; } else if (isFullRbf[tx.txid]) { diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 7a8ab3f06..ad97d5f3d 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -173,7 +173,8 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf'; + rate?: number; // effective fee rate + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf'; context?: 'projected' | 'actual'; } diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index 991fe2680..15d97fa8d 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -89,7 +89,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf'; context?: 'projected' | 'actual'; } From 565336df21a6d2c0a821e73752aee9d8bf8c36be Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 16 Jul 2023 18:39:51 +0900 Subject: [PATCH 16/18] Set missing websocket init data --- backend/src/api/websocket-handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 48e9106f0..e31221dfd 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -236,7 +236,7 @@ class WebsocketHandler { } if (parsedMessage.action === 'init') { - if (!this.socketData['blocks']?.length || !this.socketData['da']) { + if (!this.socketData['blocks']?.length || !this.socketData['da'] || !this.socketData['backendInfo'] || !this.socketData['conversions']) { this.updateSocketData(); } if (!this.socketData['blocks']?.length) { From a7ec9138c3308e692625550396b004b119c6a780 Mon Sep 17 00:00:00 2001 From: wiz Date: Mon, 17 Jul 2023 01:14:37 +0900 Subject: [PATCH 17/18] ops: Bump elements tag to 22.1.1 --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index 1121f5b4f..9ea9b7a75 100755 --- a/production/install +++ b/production/install @@ -353,7 +353,7 @@ ELEMENTS_REPO_URL=https://github.com/ElementsProject/elements ELEMENTS_REPO_NAME=elements ELEMENTS_REPO_BRANCH=master #ELEMENTS_LATEST_RELEASE=$(curl -s https://api.github.com/repos/ElementsProject/elements/releases/latest|grep tag_name|head -1|cut -d '"' -f4) -ELEMENTS_LATEST_RELEASE=elements-22.1 +ELEMENTS_LATEST_RELEASE=elements-22.1.1 echo -n '.' BITCOIN_ELECTRS_REPO_URL=https://github.com/mempool/electrs From bf5a16b043cb5eb7b70d396bd25c4fd3fcbe649c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 17 Jul 2023 11:02:28 +0900 Subject: [PATCH 18/18] always send 6 latest transactions to websocket clients --- backend/src/api/websocket-handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index e31221dfd..ab7dcf443 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -419,7 +419,7 @@ class WebsocketHandler { memPool.addToSpendMap(newTransactions); const recommendedFees = feeApi.getRecommendedFee(); - const latestTransactions = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx)); + const latestTransactions = memPool.getLatestTransactions(); // update init data const socketDataFields = {