From 5411eb491f9326425aa7d6a72f5186b0d2d9691e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 7 Jan 2024 17:57:36 +0000 Subject: [PATCH] Limit GBT: fix candidate set inconsistency --- backend/src/api/mempool-blocks.ts | 27 ++++++++++++++++----------- backend/src/api/mempool.ts | 14 ++++++++++---- backend/src/api/websocket-handler.ts | 4 ++-- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index fd9918f71..2308f6671 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -91,7 +91,7 @@ class MempoolBlocks { // set missing short ids for (const txid of transactions) { const tx = newMempool[txid]; - this.setUid(tx, false); + this.setUid(tx, !saveResults); } const accelerations = useAccelerations ? mempool.getAccelerations() : {}; @@ -170,7 +170,7 @@ class MempoolBlocks { for (const tx of addedAndChanged) { this.setUid(tx, false); } - const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[]; + const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[]; // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread @@ -196,10 +196,10 @@ class MempoolBlocks { }); this.txSelectionWorker?.once('error', reject); }); - this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedUids }); + this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedTxs.map(tx => tx.uid) as number[] }); const { blocks, rates, clusters } = this.convertResultTxids(await workerResultPromise); - this.removeUids(removedUids); + this.removeUids(removedTxs); // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); @@ -227,7 +227,7 @@ class MempoolBlocks { const transactions = txids.map(txid => newMempool[txid]).filter(tx => tx != null); // set missing short ids for (const tx of transactions) { - this.setUid(tx, false); + this.setUid(tx, !saveResults); } // set short ids for transaction inputs for (const tx of transactions) { @@ -237,6 +237,7 @@ class MempoolBlocks { const accelerations = useAccelerations ? mempool.getAccelerations() : {}; const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); const convertedAccelerations = acceleratedList.map(acc => { + this.setUid(newMempool[acc.txid], true); return { uid: this.getUid(newMempool[acc.txid]), delta: acc.feeDelta, @@ -292,11 +293,12 @@ class MempoolBlocks { for (const tx of added) { tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[]; } - const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[]; + const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[]; const accelerations = useAccelerations ? mempool.getAccelerations() : {}; const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); const convertedAccelerations = acceleratedList.map(acc => { + this.setUid(newMempool[acc.txid], true); return { uid: this.getUid(newMempool[acc.txid]), delta: acc.feeDelta, @@ -308,7 +310,7 @@ class MempoolBlocks { const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids( await this.rustGbtGenerator.update( added as RustThreadTransaction[], - removedUids, + removedTxs.map(tx => tx.uid) as number[], convertedAccelerations as RustThreadAcceleration[], this.nextUid, ), @@ -319,7 +321,7 @@ class MempoolBlocks { throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`); } else { const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true); - this.removeUids(removedUids); + this.removeUids(removedTxs); logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } @@ -522,9 +524,12 @@ class MempoolBlocks { } } - private removeUids(uids: number[]): void { - for (const uid of uids) { - this.uidMap.delete(uid); + private removeUids(txs: MempoolTransactionExtended[]): void { + for (const tx of txs) { + if (tx.uid != null) { + this.uidMap.delete(tx.uid); + tx.uid = undefined; + } } } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index b08848b26..68f91b5b0 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -320,8 +320,6 @@ class Mempool { }, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES); } - const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip); - const deletedTransactions: MempoolTransactionExtended[] = []; if (this.mempoolProtection !== 1) { @@ -341,6 +339,8 @@ class Mempool { } } + const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip, deletedTransactions); + const newMempoolSize = currentMempoolSize + newTransactions.length - deletedTransactions.length; const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); @@ -445,8 +445,12 @@ class Mempool { } } - public async getNextCandidates(minFeeTransactions: string[], blockHeight: number): Promise { + public async getNextCandidates(minFeeTransactions: string[], blockHeight: number, deletedTransactions: MempoolTransactionExtended[]): Promise { if (this.limitGBT) { + const deletedTxsMap = {}; + for (const tx of deletedTransactions) { + deletedTxsMap[tx.txid] = tx; + } const newCandidateTxMap = {}; for (const txid of minFeeTransactions) { if (this.mempoolCache[txid]) { @@ -463,14 +467,16 @@ class Mempool { } else { for (const txid of Object.keys(this.mempoolCandidates)) { if (!newCandidateTxMap[txid]) { - removed.push(this.mempoolCache[txid]); if (this.mempoolCache[txid]) { + removed.push(this.mempoolCache[txid]); this.mempoolCache[txid].effectiveFeePerVsize = this.mempoolCache[txid].adjustedFeePerVsize; this.mempoolCache[txid].ancestors = []; this.mempoolCache[txid].descendants = []; this.mempoolCache[txid].bestDescendant = null; this.mempoolCache[txid].cpfpChecked = false; this.mempoolCache[txid].cpfpUpdated = undefined; + } else if (deletedTxsMap[txid]) { + removed.push(deletedTxsMap[txid]); } } } diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index d42fc11f0..63e7f383c 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -750,7 +750,7 @@ class WebsocketHandler { const _memPool = memPool.getMempool(); const candidateTxs = await memPool.getMempoolCandidates(); let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined; - let transactionIds: string[] = (memPool.limitGBT && candidates) ? Object.keys(_memPool) : Object.keys(candidates?.txs || {}); + let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool); const accelerations = Object.values(mempool.getAccelerations()); await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, transactions); @@ -839,7 +839,7 @@ class WebsocketHandler { if (memPool.limitGBT) { const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null; const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1; - candidates = await memPool.getNextCandidates(minFeeMempool, minFeeTip); + candidates = await memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions); transactionIds = Object.keys(candidates?.txs || {}); } else { candidates = undefined;