From 702ff2796acd688d12a08020e5492752243e3dc8 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 10 Oct 2022 22:13:04 +0000 Subject: [PATCH] New projected block transaction selection algo --- backend/src/api/mempool-blocks.ts | 229 ++++++++++++++++++++++++++- backend/src/api/websocket-handler.ts | 22 ++- backend/src/mempool.interfaces.ts | 11 +- 3 files changed, 254 insertions(+), 8 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 5eb5aa9c8..9b58f4754 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,5 +1,5 @@ import logger from '../logger'; -import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces'; +import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, TransactionSet, Ancestor } from '../mempool.interfaces'; import { Common } from './common'; import config from '../config'; @@ -99,6 +99,7 @@ class MempoolBlocks { if (transactions.length) { mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length)); } + // Calculate change from previous block states for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) { let added: TransactionStripped[] = []; @@ -132,12 +133,238 @@ class MempoolBlocks { removed }); } + return { blocks: mempoolBlocks, deltas: mempoolBlockDeltas }; } + /* + * Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core + * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) + * + * templateLimit: number of blocks to build using the full algo, + * remaining blocks up to blockLimit will skip the expensive updateDescendants step + * + * blockLimit: number of blocks to build in total. Excess transactions will be ignored. + */ + public makeBlockTemplates(mempool: { [txid: string]: TransactionExtended }, templateLimit: number = Infinity, blockLimit: number = Infinity): MempoolBlockWithTransactions[] { + const start = new Date().getTime(); + const txSets: { [txid: string]: TransactionSet } = {}; + const mempoolArray: TransactionExtended[] = Object.values(mempool); + + mempoolArray.forEach((tx) => { + tx.bestDescendant = null; + tx.ancestors = []; + tx.cpfpChecked = false; + tx.effectiveFeePerVsize = tx.feePerVsize; + txSets[tx.txid] = { + fee: 0, + weight: 1, + score: 0, + children: [], + available: true, + modified: false, + }; + }); + + // Build relatives graph & calculate ancestor scores + mempoolArray.forEach((tx) => { + this.setRelatives(tx, mempool, txSets); + }); + + // Sort by descending ancestor score + const byAncestor = (a, b): number => this.sortByAncestorScore(a, b, txSets); + mempoolArray.sort(byAncestor); + + // Build blocks by greedily choosing the highest feerate package + // (i.e. the package rooted in the transaction with the best ancestor score) + const blocks: MempoolBlockWithTransactions[] = []; + let blockWeight = 4000; + let blockSize = 0; + let transactions: TransactionExtended[] = []; + let modified: TransactionExtended[] = []; + let overflow: TransactionExtended[] = []; + let failures = 0; + while ((mempoolArray.length || modified.length) && blocks.length < blockLimit) { + const simpleMode = blocks.length >= templateLimit; + let anyModified = false; + // Select best next package + let nextTx; + if (mempoolArray.length && (!modified.length || txSets[mempoolArray[0].txid]?.score > txSets[modified[0].txid]?.score)) { + nextTx = mempoolArray.shift(); + if (txSets[nextTx?.txid]?.modified) { + nextTx = null; + } + } else { + nextTx = modified.shift(); + } + + if (nextTx && txSets[nextTx.txid]?.available) { + const nextTxSet = txSets[nextTx.txid]; + // Check if the package fits into this block + if (nextTxSet && blockWeight + nextTxSet.weight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) { + blockWeight += nextTxSet.weight; + // sort txSet by dependency graph (equivalent to sorting by ascending ancestor count) + const sortedTxSet = nextTx.ancestors.sort((a, b) => { + return (mempool[a.txid]?.ancestors?.length || 0) - (mempool[b.txid]?.ancestors?.length || 0); + }); + [...sortedTxSet, nextTx].forEach((ancestor, i, arr) => { + const tx = mempool[ancestor.txid]; + const txSet = txSets[ancestor.txid]; + if (txSet.available) { + txSet.available = false; + tx.effectiveFeePerVsize = nextTxSet.fee / (nextTxSet.weight / 4); + tx.cpfpChecked = true; + if (i < arr.length - 1) { + tx.bestDescendant = { + txid: arr[i + 1].txid, + fee: arr[i + 1].fee, + weight: arr[i + 1].weight, + }; + } + transactions.push(tx); + blockSize += tx.size; + } + }); + + // remove these as valid package ancestors for any remaining descendants + if (!simpleMode) { + sortedTxSet.forEach(tx => { + anyModified = this.updateDescendants(tx, tx, mempool, txSets, modified); + }); + } + + failures = 0; + } else { + // hold this package in an overflow list while we check for smaller options + txSets[nextTx.txid].modified = true; + overflow.push(nextTx); + failures++; + } + } + + // this block is full + const outOfTransactions = !mempoolArray.length && !modified.length; + const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000); + const exceededSimpleTries = failures > 0 && simpleMode; + if (outOfTransactions || exceededPackageTries || exceededSimpleTries) { + // construct this block + blocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, blocks.length)); + // reset for the next block + transactions = []; + blockSize = 0; + blockWeight = 4000; + + // 'overflow' packages didn't fit in this block, but are valid candidates for the next + if (overflow.length) { + modified = modified.concat(overflow); + overflow = []; + anyModified = true; + } + } + + // re-sort modified list if necessary + if (anyModified) { + modified = modified.filter(tx => txSets[tx.txid]?.available).sort(byAncestor); + } + } + + const end = new Date().getTime(); + const time = end - start; + logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds'); + + return blocks; + } + + private sortByAncestorScore(a, b, txSets): number { + return txSets[b.txid]?.score - txSets[a.txid]?.score; + } + + private setRelatives(tx: TransactionExtended, mempool: { [txid: string]: TransactionExtended }, txSets: { [txid: string]: TransactionSet }): { [txid: string]: Ancestor } { + let ancestors: { [txid: string]: Ancestor } = {}; + tx.vin.forEach((parent) => { + const parentTx = mempool[parent.txid]; + const parentTxSet = txSets[parent.txid]; + if (parentTx && parentTxSet) { + ancestors[parentTx.txid] = parentTx; + if (!parentTxSet.children) { + parentTxSet.children = [tx.txid]; + } else { + parentTxSet.children.push(tx.txid); + } + if (!parentTxSet.score) { + ancestors = { + ...ancestors, + ...this.setRelatives(parentTx, mempool, txSets), + }; + } + } + }); + tx.ancestors = Object.values(ancestors).map(ancestor => { + return { + txid: ancestor.txid, + fee: ancestor.fee, + weight: ancestor.weight + }; + }); + let totalFees = tx.fee; + let totalWeight = tx.weight; + tx.ancestors.forEach(ancestor => { + totalFees += ancestor.fee; + totalWeight += ancestor.weight; + }); + txSets[tx.txid].fee = totalFees; + txSets[tx.txid].weight = totalWeight; + txSets[tx.txid].score = this.calcAncestorScore(tx, totalFees, totalWeight); + + return ancestors; + } + + private calcAncestorScore(tx: TransactionExtended, ancestorFees: number, ancestorWeight: number): number { + return Math.min(tx.fee / tx.weight, ancestorFees / ancestorWeight); + } + + // walk over remaining descendants, removing the root as a valid ancestor & updating the ancestor score + // returns whether any descendants were modified + private updateDescendants( + root: TransactionExtended, + tx: TransactionExtended, + mempool: { [txid: string]: TransactionExtended }, + txSets: { [txid: string]: TransactionSet }, + modified: TransactionExtended[], + ): boolean { + let anyModified = false; + const txSet = txSets[tx.txid]; + if (txSet.children) { + txSet.children.forEach(childId => { + const child = mempool[childId]; + if (child && child.ancestors && txSets[childId]?.available) { + const ancestorIndex = child.ancestors.findIndex(a => a.txid === root.txid); + if (ancestorIndex > -1) { + // remove tx as ancestor + child.ancestors.splice(ancestorIndex, 1); + const childTxSet = txSets[childId]; + childTxSet.fee -= root.fee; + childTxSet.weight -= root.weight; + childTxSet.score = this.calcAncestorScore(child, childTxSet.fee, childTxSet.weight); + anyModified = true; + + if (!childTxSet.modified) { + childTxSet.modified = true; + modified.push(child); + } + + // recursively update grandchildren + anyModified = this.updateDescendants(root, child, mempool, txSets, modified) || anyModified; + } + } + }); + } + return anyModified; + } + private dataToMempoolBlocks(transactions: TransactionExtended[], blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions { let rangeLength = 4; diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 4896ee058..f183a4799 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -414,15 +414,15 @@ class WebsocketHandler { let mBlockDeltas: undefined | MempoolBlockDelta[]; let matchRate = 0; const _memPool = memPool.getMempool(); - const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + const projectedBlocks = mempoolBlocks.makeBlockTemplates(cloneMempool(_memPool), 1, 1); - if (_mempoolBlocks[0]) { + if (projectedBlocks[0]) { const matches: string[] = []; const added: string[] = []; const missing: string[] = []; for (const txId of txIds) { - if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) { + if (projectedBlocks[0].transactionIds.indexOf(txId) > -1) { matches.push(txId); } else { added.push(txId); @@ -430,7 +430,7 @@ class WebsocketHandler { delete _memPool[txId]; } - for (const txId of _mempoolBlocks[0].transactionIds) { + for (const txId of projectedBlocks[0].transactionIds) { if (matches.includes(txId) || added.includes(txId)) { continue; } @@ -443,14 +443,14 @@ class WebsocketHandler { mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); if (Common.indexingEnabled()) { - const stripped = _mempoolBlocks[0].transactions.map((tx) => { + const stripped = projectedBlocks[0].transactions.map((tx) => { return { txid: tx.txid, vsize: tx.vsize, fee: tx.fee ? Math.round(tx.fee) : 0, value: tx.value, }; - }); + }); BlocksSummariesRepository.$saveSummary({ height: block.height, template: { @@ -580,4 +580,14 @@ class WebsocketHandler { } } +function cloneMempool(mempool: { [txid: string]: TransactionExtended }): { [txid: string]: TransactionExtended } { + const cloned = {}; + Object.keys(mempool).forEach(id => { + cloned[id] = { + ...mempool[id] + }; + }); + return cloned; +} + export default new WebsocketHandler(); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index d72b13576..fec26c0f3 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -70,12 +70,21 @@ export interface TransactionExtended extends IEsploraApi.Transaction { deleteAfter?: number; } -interface Ancestor { +export interface Ancestor { txid: string; weight: number; fee: number; } +export interface TransactionSet { + fee: number; + weight: number; + score: number; + children?: string[]; + available?: boolean; + modified?: boolean; +} + interface BestDescendant { txid: string; weight: number;