From e2d3bb4cc571ccff4e20512620aa84477096b777 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 17 Feb 2023 17:54:29 -0600 Subject: [PATCH] Use minfee node to limit gbt input size --- backend/src/api/mempool-blocks.ts | 89 +++++++++++++++++----------- backend/src/api/mempool.ts | 70 +++++++++++++++++++--- backend/src/api/websocket-handler.ts | 60 +++++++++++++++---- backend/src/config.ts | 2 + backend/src/index.ts | 5 +- backend/src/mempool.interfaces.ts | 6 ++ 6 files changed, 176 insertions(+), 56 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index b9da7d4e8..09c5d8e79 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,6 +1,6 @@ import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt'; import logger from '../logger'; -import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag, TransactionClassified, TransactionCompressed, MempoolDeltaChange } from '../mempool.interfaces'; +import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; @@ -40,12 +40,9 @@ class MempoolBlocks { return this.mempoolBlockDeltas; } - public updateMempoolBlocks(memPool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): MempoolBlockWithTransactions[] { + public updateMempoolBlocks(transactions: string[], memPool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false): MempoolBlockWithTransactions[] { const latestMempool = memPool; - const memPoolArray: MempoolTransactionExtended[] = []; - for (const i in latestMempool) { - memPoolArray.push(latestMempool[i]); - } + const memPoolArray: MempoolTransactionExtended[] = transactions.map(txid => memPool[txid]); const start = new Date().getTime(); // Clear bestDescendants & ancestors @@ -207,7 +204,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { + public async $makeBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { const start = Date.now(); // reset mempool short ids @@ -215,8 +212,9 @@ class MempoolBlocks { this.resetUids(); } // set missing short ids - for (const tx of Object.values(newMempool)) { - this.setUid(tx, !saveResults); + for (const txid of transactions) { + const tx = newMempool[txid]; + this.setUid(tx, false); } const accelerations = useAccelerations ? mempool.getAccelerations() : {}; @@ -224,7 +222,8 @@ class MempoolBlocks { // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread const strippedMempool: Map = new Map(); - Object.values(newMempool).forEach(entry => { + for (const txid of transactions) { + const entry = newMempool[txid]; if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, @@ -237,7 +236,7 @@ class MempoolBlocks { }; strippedMempool.set(entry.uid, stripped); } - }); + } // (re)initialize tx selection worker thread if (!this.txSelectionWorker) { @@ -268,7 +267,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, accelerationPool, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), candidates, accelerations, accelerationPool, saveResults); logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -279,10 +278,10 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise { + public async $updateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise { if (!this.txSelectionWorker) { // need to reset the worker - await this.$makeBlockTemplates(newMempool, saveResults, useAccelerations); + await this.$makeBlockTemplates(transactions, newMempool, candidates, saveResults, useAccelerations); return; } @@ -292,7 +291,7 @@ class MempoolBlocks { const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added; for (const tx of addedAndChanged) { - this.setUid(tx, true); + this.setUid(tx, false); } const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[]; @@ -328,7 +327,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, null, saveResults); + this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), candidates, accelerations, null, saveResults); logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`); } catch (e) { logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); @@ -336,23 +335,26 @@ class MempoolBlocks { } private resetRustGbt(): void { + console.log('reset rust gbt'); this.rustInitialized = false; this.rustGbtGenerator = new GbtGenerator(); } - public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { + public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { const start = Date.now(); // reset mempool short ids if (saveResults) { this.resetUids(); } + + const transactions = txids.map(txid => newMempool[txid]).filter(tx => tx != null); // set missing short ids - for (const tx of Object.values(newMempool)) { - this.setUid(tx, !saveResults); + for (const tx of transactions) { + this.setUid(tx, false); } // set short ids for transaction inputs - for (const tx of Object.values(newMempool)) { + for (const tx of transactions) { tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[]; } @@ -369,15 +371,15 @@ class MempoolBlocks { const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator(); try { const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids( - await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid), + await rustGbt.make(transactions as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid), ); if (saveResults) { this.rustInitialized = true; } - const mempoolSize = Object.keys(newMempool).length; + const expectedSize = transactions.length; const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length; - logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`); - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults); + logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${expectedSize} in the mempool, ${overflow.length} were unmineable`); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -389,26 +391,26 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, useAccelerations: boolean, accelerationPool?: number): Promise { - return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool); + public async $oneOffRustBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise { + return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, false, useAccelerations, accelerationPool); } - public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise { + public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise { // GBT optimization requires that uids never get too sparse // as a sanity check, we should also explicitly prevent uint32 uid overflow - if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) { + if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * transactions.length), MAX_UINT32)) { this.resetRustGbt(); } if (!this.rustInitialized) { // need to reset the worker - return this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool); + return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, true, useAccelerations, accelerationPool); } const start = Date.now(); // set missing short ids for (const tx of added) { - this.setUid(tx, true); + this.setUid(tx, false); } // set short ids for transaction inputs for (const tx of added) { @@ -416,6 +418,8 @@ class MempoolBlocks { } const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[]; + console.log(`removing ${removed.length} (${removedUids.length} with uids)`); + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); const convertedAccelerations = acceleratedList.map(acc => { @@ -425,6 +429,8 @@ class MempoolBlocks { }; }); + console.log(`${acceleratedList.length} accelerations`); + // run the block construction algorithm in a separate thread, and wait for a result try { const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids( @@ -436,11 +442,11 @@ class MempoolBlocks { ), ); const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length; - logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`); - if (mempoolSize !== resultMempoolSize) { - throw new Error('GBT returned wrong number of transactions , cache is probably out of sync'); + logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${transactions.length} candidates, ${overflow.length} were unmineable`); + if (transactions.length !== resultMempoolSize) { + throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`); } else { - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true); this.removeUids(removedUids); logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; @@ -452,7 +458,7 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] { + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] { for (const [txid, rate] of rates) { if (txid in mempool) { mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize); @@ -486,6 +492,9 @@ class MempoolBlocks { if (txid === memberTxid) { matched = true; } else { + if (!mempool[txid]) { + console.log('txid missing from mempool! ', txid, candidates?.txs[txid]); + } const relative = { txid: txid, fee: mempool[txid].fee, @@ -518,6 +527,16 @@ class MempoolBlocks { let totalWeight = 0; let totalFees = 0; const transactions: MempoolTransactionExtended[] = []; + + // backfill purged transactions + if (candidates?.txs && blockIndex === blocks.length - 1) { + for (const txid of Object.keys(mempool)) { + if (!candidates.txs[txid]) { + block.push(txid); + } + } + } + for (const txid of block) { if (txid) { mempoolTx = mempool[txid]; diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 4c2f248ac..540e0828b 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,6 +1,6 @@ import config from '../config'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; -import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; +import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond, GbtCandidates } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; import transactionUtils from './transaction-utils'; @@ -11,18 +11,21 @@ import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import rbfCache from './rbf-cache'; import { Acceleration } from './services/acceleration'; import redisCache from './redis-cache'; +import blocks from './blocks'; class Mempool { private inSync: boolean = false; private mempoolCacheDelta: number = -1; private mempoolCache: { [txId: string]: MempoolTransactionExtended } = {}; + private mempoolCandidates: { [txid: string ]: boolean } = {}; + private minFeeMempool: { [txId: string]: boolean } = {}; private spendMap = new Map(); private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0, maxmempool: 300000000, mempoolminfee: Common.isLiquid() ? 0.00000100 : 0.00001000, minrelaytxfee: Common.isLiquid() ? 0.00000100 : 0.00001000 }; private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined; private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], - deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise) | undefined; + deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[], candidates?: GbtCandidates) => Promise) | undefined; private accelerations: { [txId: string]: Acceleration } = {}; @@ -40,6 +43,8 @@ class Mempool { private missingTxCount = 0; private mainLoopTimeout: number = 120000; + public limitGBT = config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE && config.MEMPOOL.LIMIT_GBT; + constructor() { setInterval(this.updateTxPerSecond.bind(this), 1000); } @@ -74,7 +79,8 @@ class Mempool { } public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise): void { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[], + candidates?: GbtCandidates) => Promise): void { this.$asyncMempoolChangedCallback = fn; } @@ -117,7 +123,7 @@ class Mempool { this.mempoolChangedCallback(this.mempoolCache, [], [], []); } if (this.$asyncMempoolChangedCallback) { - await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], []); + await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], [], this.limitGBT ? { txs: {}, added: [], removed: [] } : undefined); } this.addToSpendMap(Object.values(this.mempoolCache)); } @@ -160,6 +166,10 @@ class Mempool { return newTransactions; } + public getMempoolCandidates(): { [txid: string]: boolean } { + return this.mempoolCandidates; + } + public async $updateMemPoolInfo() { this.mempoolInfo = await this.$getMempoolInfo(); } @@ -189,7 +199,7 @@ class Mempool { return txTimes; } - public async $updateMempool(transactions: string[], accelerations: Acceleration[] | null, pollRate: number): Promise { + public async $updateMempool(transactions: string[], accelerations: Acceleration[] | null, minFeeMempool: string[], minFeeTip: number, pollRate: number): Promise { logger.debug(`Updating mempool...`); // warn if this run stalls the main loop for more than 2 minutes @@ -311,6 +321,8 @@ class Mempool { }, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES); } + const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip); + const deletedTransactions: MempoolTransactionExtended[] = []; if (this.mempoolProtection !== 1) { @@ -341,12 +353,14 @@ class Mempool { this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize); + const candidatesChanged = candidates?.added?.length || candidates?.removed?.length; + if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta); } - if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) { + if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length || candidatesChanged)) { this.updateTimerProgress(timer, 'running async mempool callback'); - await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta); + await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta, candidates); this.updateTimerProgress(timer, 'completed async mempool callback'); } @@ -432,6 +446,48 @@ class Mempool { } } + public async getNextCandidates(minFeeTransactions: string[], blockHeight: number): Promise { + if (this.limitGBT) { + const newCandidateTxMap = {}; + this.minFeeMempool = {}; + for (const txid of minFeeTransactions) { + if (this.mempoolCache[txid]) { + newCandidateTxMap[txid] = true; + } + this.minFeeMempool[txid] = true; + } + const removed: MempoolTransactionExtended[] = []; + const added: MempoolTransactionExtended[] = []; + // don't prematurely remove txs included in a new block + if (blockHeight > blocks.getCurrentBlockHeight()) { + for (const txid of Object.keys(this.mempoolCandidates)) { + newCandidateTxMap[txid] = true; + } + } else { + for (const txid of Object.keys(this.mempoolCandidates)) { + if (!newCandidateTxMap[txid]) { + const tx = this.mempoolCache[txid]; + removed.push(tx); + } + } + } + + for (const txid of Object.keys(newCandidateTxMap)) { + if (!this.mempoolCandidates[txid]) { + const tx = this.mempoolCache[txid]; + added.push(tx); + } + } + + this.mempoolCandidates = newCandidateTxMap; + return { + txs: this.mempoolCandidates, + added, + removed + }; + } + } + private startTimer() { const state: any = { start: Date.now(), diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 2884f72eb..ac79d828a 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -2,7 +2,7 @@ import logger from '../logger'; import * as WebSocket from 'ws'; import { BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse, - OptimizedStatistic, ILoadingIndicators + OptimizedStatistic, ILoadingIndicators, GbtCandidates, } from '../mempool.interfaces'; import blocks from './blocks'; import memPool from './mempool'; @@ -32,6 +32,7 @@ interface AddressTransactions { confirmed: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], } +import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; // valid 'want' subscriptions const wantable = [ @@ -436,21 +437,33 @@ class WebsocketHandler { } async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]): Promise { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[], + candidates?: GbtCandidates): Promise { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } this.printLogs(); + const transactionIds = (memPool.limitGBT && candidates) ? Object.keys(candidates?.txs || {}) : Object.keys(newMempool); + let added = newTransactions; + let removed = deletedTransactions; + console.log(`handleMempoolChange: ${newTransactions.length} new txs, ${deletedTransactions.length} removed, ${candidates?.added.length} candidate added, ${candidates?.removed.length} candidate removed`); + console.log(`mempool size ${Object.keys(newMempool).length}, candidates: ${transactionIds.length}`); + if (memPool.limitGBT) { + console.log('GBT on limited mempool...'); + added = candidates?.added || []; + removed = candidates?.removed || []; + } + if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, config.MEMPOOL_SERVICES.ACCELERATIONS); + await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, config.MEMPOOL_SERVICES.ACCELERATIONS); } else { - await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); + await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } } else { - mempoolBlocks.updateMempoolBlocks(newMempool, true); + mempoolBlocks.updateMempoolBlocks(transactionIds, newMempool, candidates, true); } const mBlocks = mempoolBlocks.getMempoolBlocks(); @@ -739,6 +752,9 @@ class WebsocketHandler { await statistics.runStatistics(); const _memPool = memPool.getMempool(); + const candidateTxs = await memPool.getMempoolCandidates(); + let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined; + let transactionIds: string[] = (memPool.limitGBT && candidates) ? Object.keys(_memPool) : Object.keys(candidates?.txs || {}); const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); @@ -759,19 +775,23 @@ class WebsocketHandler { auditMempool = deepClone(_memPool); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { if (config.MEMPOOL.RUST_GBT) { - projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool, isAccelerated, block.extras.pool.id); + projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(transactionIds, auditMempool, candidates, isAccelerated, block.extras.pool.id); } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id); } } else { - projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); + projectedBlocks = mempoolBlocks.updateMempoolBlocks(transactionIds, auditMempool, candidates, false); } } else { if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) { if (config.MEMPOOL.RUST_GBT) { - projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id); + console.log(`handleNewBlock: ${transactions.length} mined txs, ${candidates?.added.length} candidate added, ${candidates?.removed.length} candidate removed`); + console.log(`mempool size ${Object.keys(auditMempool).length}, candidates: ${transactionIds.length}`); + let added = memPool.limitGBT ? (candidates?.added || []) : []; + let removed = memPool.limitGBT ? (candidates?.removed || []) : []; + projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id); } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id); } } else { projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); @@ -838,14 +858,28 @@ class WebsocketHandler { confirmedTxids[txId] = true; } + if (memPool.limitGBT) { + const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null; + const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1; + candidates = await memPool.getNextCandidates(minFeeMempool, minFeeTip); + transactionIds = Object.keys(candidates?.txs || {}); + } else { + candidates = undefined; + transactionIds = Object.keys(memPool.getMempool()); + } + if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions, true); + console.log(`handleNewBlock (after): ${transactions.length} mined txs, ${candidates?.added.length} candidate added, ${candidates?.removed.length} candidate removed`); + console.log(`mempool size ${Object.keys(_memPool).length}, candidates: ${transactionIds.length}`); + let added = memPool.limitGBT ? (candidates?.added || []) : []; + let removed = memPool.limitGBT ? (candidates?.removed || []) : transactions; + await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true); } else { - await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); + await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } } else { - mempoolBlocks.updateMempoolBlocks(_memPool, true); + mempoolBlocks.updateMempoolBlocks(transactionIds, _memPool, candidates, true); } const mBlocks = mempoolBlocks.getMempoolBlocks(); const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); diff --git a/backend/src/config.ts b/backend/src/config.ts index ee3645e58..6a202a392 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -35,6 +35,7 @@ interface IConfig { ADVANCED_GBT_AUDIT: boolean; ADVANCED_GBT_MEMPOOL: boolean; RUST_GBT: boolean; + LIMIT_GBT: boolean; CPFP_INDEXING: boolean; MAX_BLOCKS_BULK_QUERY: number; DISK_CACHE_BLOCK_INTERVAL: number; @@ -197,6 +198,7 @@ const defaults: IConfig = { 'ADVANCED_GBT_AUDIT': false, 'ADVANCED_GBT_MEMPOOL': false, 'RUST_GBT': false, + 'LIMIT_GBT': false, 'CPFP_INDEXING': false, 'MAX_BLOCKS_BULK_QUERY': 0, 'DISK_CACHE_BLOCK_INTERVAL': 6, diff --git a/backend/src/index.ts b/backend/src/index.ts index 1988c7c56..ae5113029 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -45,6 +45,7 @@ import { formatBytes, getBytesUnit } from './utils/format'; import redisCache from './api/redis-cache'; import accelerationApi from './api/services/acceleration'; import bitcoinCoreRoutes from './api/bitcoin/bitcoin-core.routes'; +import bitcoinSecondClient from './api/bitcoin/bitcoin-second-client'; class Server { private wss: WebSocket.Server | undefined; @@ -215,11 +216,13 @@ class Server { } } const newMempool = await bitcoinApi.$getRawMempool(); + const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null; + const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1; const newAccelerations = await accelerationApi.$fetchAccelerations(); const numHandledBlocks = await blocks.$updateBlocks(); const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerIsRunning() ? 10 : 1); if (numHandledBlocks === 0) { - await memPool.$updateMempool(newMempool, newAccelerations, pollRate); + await memPool.$updateMempool(newMempool, newAccelerations, minFeeMempool, minFeeTip, pollRate); } indexer.$run(); if (config.FIAT_PRICE.ENABLED) { diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index d04858607..64c10d322 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -143,6 +143,12 @@ export interface CompactThreadTransaction { dirty?: boolean; } +export interface GbtCandidates { + txs: { [txid: string ]: boolean }, + added: MempoolTransactionExtended[]; + removed: MempoolTransactionExtended[]; +} + export interface ThreadTransaction { txid: string; fee: number;