From f1966768a7e767ee8f611b8e821793883ce73acf Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 19 Jun 2023 18:14:09 -0400 Subject: [PATCH] exclude fullrbf txs from audit and label in visualization --- backend/src/api/audit.ts | 15 +++++++++++---- backend/src/api/database-migration.ts | 6 +++++- backend/src/api/rbf-cache.ts | 13 +++++++++++++ backend/src/api/websocket-handler.ts | 11 ++++++----- backend/src/mempool.interfaces.ts | 1 + .../src/repositories/BlocksAuditsRepository.ts | 8 +++++--- .../components/block-overview-graph/tx-view.ts | 3 ++- .../block-overview-tooltip.component.html | 1 + .../src/app/components/block/block.component.ts | 16 +++++++++++++++- .../src/app/interfaces/node-api.interface.ts | 3 ++- .../src/app/interfaces/websocket.interface.ts | 2 +- 11 files changed, 62 insertions(+), 17 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 6c5f96988..e79196a7a 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -1,19 +1,21 @@ import config from '../config'; import logger from '../logger'; import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; +import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended }) - : { censored: string[], added: string[], fresh: string[], sigop: string[], score: number, similarity: number } { + : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { - return { censored: [], added: [], fresh: [], sigop: [], score: 0, similarity: 1 }; + return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 }; } const matches: string[] = []; // present in both mined block and template const added: string[] = []; // present in mined block, not in template const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN + const fullrbf: string[] = []; // either missing or present, and part of a fullrbf replacement const isCensored = {}; // missing, without excuse const isDisplaced = {}; let displacedWeight = 0; @@ -35,7 +37,9 @@ class Audit { for (const txid of projectedBlocks[0].transactionIds) { if (!inBlock[txid]) { // tx is recent, may have reached the miner too late for inclusion - if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) { + if (rbfCache.isFullRbf(txid)) { + fullrbf.push(txid); + } else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) { fresh.push(txid); } else { isCensored[txid] = true; @@ -91,7 +95,9 @@ class Audit { if (inTemplate[tx.txid]) { matches.push(tx.txid); } else { - if (!isDisplaced[tx.txid]) { + if (rbfCache.isFullRbf(tx.txid)) { + fullrbf.push(tx.txid); + } else if (!isDisplaced[tx.txid]) { added.push(tx.txid); } overflowWeight += tx.weight; @@ -138,6 +144,7 @@ class Audit { added, fresh, sigop: [], + fullrbf, score, similarity, }; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 22b42dac7..a9266a016 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 62; + private static currentVersion = 63; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -539,6 +539,10 @@ class DatabaseMigration { await this.updateToSchemaVersion(62); } + if (databaseSchemaVersion < 63 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"'); + await this.updateToSchemaVersion(63); + } } /** diff --git a/backend/src/api/rbf-cache.ts b/backend/src/api/rbf-cache.ts index f0a916c8c..79d5ff2d1 100644 --- a/backend/src/api/rbf-cache.ts +++ b/backend/src/api/rbf-cache.ts @@ -169,6 +169,19 @@ class RbfCache { } } + // is the transaction involved in a full rbf replacement? + public isFullRbf(txid: string): boolean { + const treeId = this.treeMap.get(txid); + if (!treeId) { + return false; + } + const tree = this.rbfTrees.get(treeId); + if (!tree) { + return false; + } + return tree?.fullRbf; + } + private cleanup(): void { const now = Date.now(); for (const txid of this.expiring.keys()) { diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 97092d2b1..ae536b72e 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -583,6 +583,10 @@ class WebsocketHandler { const _memPool = memPool.getMempool(); + const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap()); + memPool.handleMinedRbfTransactions(rbfTransactions); + memPool.removeFromSpendMap(transactions); + if (config.MEMPOOL.AUDIT) { let projectedBlocks; let auditMempool = _memPool; @@ -605,7 +609,7 @@ class WebsocketHandler { } if (Common.indexingEnabled() && memPool.isInSync()) { - const { censored, added, fresh, sigop, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); + const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); const matchRate = Math.round(score * 100 * 100) / 100; const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; @@ -633,6 +637,7 @@ class WebsocketHandler { missingTxs: censored, freshTxs: fresh, sigopTxs: sigop, + fullrbfTxs: fullrbf, matchRate: matchRate, expectedFees: totalFees, expectedWeight: totalWeight, @@ -652,10 +657,6 @@ class WebsocketHandler { } } - const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap()); - memPool.handleMinedRbfTransactions(rbfTransactions); - memPool.removeFromSpendMap(transactions); - // Update mempool to remove transactions included in the new block for (const txId of txIds) { delete _memPool[txId]; diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index cc038ecfd..adcb9645d 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -34,6 +34,7 @@ export interface BlockAudit { missingTxs: string[], freshTxs: string[], sigopTxs: string[], + fullrbfTxs: string[], addedTxs: string[], matchRate: number, expectedFees?: number, diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 8ad035f32..16a4e733d 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces'; class BlocksAuditRepositories { public async $saveAudit(audit: BlockAudit): Promise { try { - await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate, expected_fees, expected_weight) - VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), - JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); + await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, match_rate, expected_fees, expected_weight) + VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), + JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`); @@ -69,6 +69,7 @@ class BlocksAuditRepositories { added_txs as addedTxs, fresh_txs as freshTxs, sigop_txs as sigopTxs, + fullrbf_txs as fullrbfTxs, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight @@ -83,6 +84,7 @@ class BlocksAuditRepositories { rows[0].addedTxs = JSON.parse(rows[0].addedTxs); rows[0].freshTxs = JSON.parse(rows[0].freshTxs); rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs); + rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs); rows[0].template = JSON.parse(rows[0].template); return rows[0]; diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index bb3d9563c..7d3e0ee13 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -37,7 +37,7 @@ export default class TxView implements TransactionStripped { value: number; feerate: number; rate?: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf'; context?: 'projected' | 'actual'; scene?: BlockScene; @@ -172,6 +172,7 @@ export default class TxView implements TransactionStripped { return auditColors.censored; case 'missing': case 'sigop': + case 'fullrbf': return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1]; case 'fresh': return auditColors.missing; diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index 795958fe3..eb6d97e40 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -48,6 +48,7 @@ Recently broadcasted Added Marginal fee rate + Full RBF diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 927222dbc..ad008089d 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -317,6 +317,7 @@ export class BlockComponent implements OnInit, OnDestroy { const isSelected = {}; const isFresh = {}; const isSigop = {}; + const isFullRbf = {}; this.numMissing = 0; this.numUnexpected = 0; @@ -339,6 +340,9 @@ export class BlockComponent implements OnInit, OnDestroy { for (const txid of blockAudit.sigopTxs || []) { isSigop[txid] = true; } + for (const txid of blockAudit.fullrbfTxs || []) { + isFullRbf[txid] = true; + } // set transaction statuses for (const tx of blockAudit.template) { tx.context = 'projected'; @@ -347,7 +351,15 @@ export class BlockComponent implements OnInit, OnDestroy { } else if (inBlock[tx.txid]) { tx.status = 'found'; } else { - tx.status = isFresh[tx.txid] ? 'fresh' : (isSigop[tx.txid] ? 'sigop' : 'missing'); + if (isFresh[tx.txid]) { + tx.status = 'fresh'; + } else if (isSigop[tx.txid]) { + tx.status = 'sigop'; + } else if (isFullRbf[tx.txid]) { + tx.status = 'fullrbf'; + } else { + tx.status = 'missing'; + } isMissing[tx.txid] = true; this.numMissing++; } @@ -360,6 +372,8 @@ export class BlockComponent implements OnInit, OnDestroy { tx.status = 'added'; } else if (inTemplate[tx.txid]) { tx.status = 'found'; + } else if (isFullRbf[tx.txid]) { + tx.status = 'fullrbf'; } else { tx.status = 'selected'; isSelected[tx.txid] = true; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 68c45b3b2..82e1ae50d 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -156,6 +156,7 @@ export interface BlockAudit extends BlockExtended { addedTxs: string[], freshTxs: string[], sigopTxs: string[], + fullrbfTxs: string[], matchRate: number, expectedFees: number, expectedWeight: number, @@ -171,7 +172,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf'; context?: 'projected' | 'actual'; } diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index 41643fb73..20a114c72 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -77,7 +77,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf'; context?: 'projected' | 'actual'; }