From 5065fa42d022bd500f360746187a1a51e23e8a4d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 26 Jun 2023 14:22:10 -0400 Subject: [PATCH] calculate total block weights inside rust gbt --- backend/rust-gbt/index.d.ts | 10 ++- backend/rust-gbt/src/gbt.rs | 4 + backend/rust-gbt/src/lib.rs | 8 +- backend/src/api/mempool-blocks.ts | 134 ++++++++++++++++-------------- 4 files changed, 87 insertions(+), 69 deletions(-) diff --git a/backend/rust-gbt/index.d.ts b/backend/rust-gbt/index.d.ts index c2c4854ee..3ddcff18b 100644 --- a/backend/rust-gbt/index.d.ts +++ b/backend/rust-gbt/index.d.ts @@ -22,13 +22,15 @@ export class GbtGenerator { * The result from calling the gbt function. * * This tuple contains the following: - * blocks: A 2D Vector of transaction IDs (u32), the inner Vecs each represent a block. - * clusters: A 2D Vector of transaction IDs representing clusters of dependent mempool transactions - * rates: A Vector of tuples containing transaction IDs (u32) and effective fee per vsize (f64) + * blocks: A 2D Vector of transaction IDs (u32), the inner Vecs each represent a block. + * block_weights: A Vector of total weights per block. + * clusters: A 2D Vector of transaction IDs representing clusters of dependent mempool transactions + * rates: A Vector of tuples containing transaction IDs (u32) and effective fee per vsize (f64) */ export class GbtResult { blocks: Array> + blockWeights: Array clusters: Array> rates: Array> - constructor(blocks: Array>, clusters: Array>, rates: Array>) + constructor(blocks: Array>, blockWeights: Array, clusters: Array>, rates: Array>) } diff --git a/backend/rust-gbt/src/gbt.rs b/backend/rust-gbt/src/gbt.rs index 0226a42ef..1899650d9 100644 --- a/backend/rust-gbt/src/gbt.rs +++ b/backend/rust-gbt/src/gbt.rs @@ -62,6 +62,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap) -> GbtResult { let mut audit_pool: AuditPool = u32hashmap_with_capacity(STARTING_CAPACITY); let mut mempool_stack: Vec = Vec::with_capacity(STARTING_CAPACITY); let mut clusters: Vec> = Vec::new(); + let mut block_weights: Vec = Vec::new(); info!("Initializing working structs"); for (uid, tx) in mempool { @@ -192,6 +193,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap) -> GbtResult { // finalize this block if !transactions.is_empty() { blocks.push(transactions); + block_weights.push(block_weight); } // reset for the next block transactions = Vec::with_capacity(STARTING_CAPACITY); @@ -221,6 +223,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap) -> GbtResult { info!("add the final unbounded block if it contains any transactions"); if !transactions.is_empty() { blocks.push(transactions); + block_weights.push(block_weight); } info!("make a list of dirty transactions and their new rates"); @@ -238,6 +241,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap) -> GbtResult { GbtResult { blocks, + block_weights, clusters, rates, } diff --git a/backend/rust-gbt/src/lib.rs b/backend/rust-gbt/src/lib.rs index f74b218f5..1696d4351 100644 --- a/backend/rust-gbt/src/lib.rs +++ b/backend/rust-gbt/src/lib.rs @@ -109,12 +109,14 @@ impl GbtGenerator { /// The result from calling the gbt function. /// /// This tuple contains the following: -/// blocks: A 2D Vector of transaction IDs (u32), the inner Vecs each represent a block. -/// clusters: A 2D Vector of transaction IDs representing clusters of dependent mempool transactions -/// rates: A Vector of tuples containing transaction IDs (u32) and effective fee per vsize (f64) +/// blocks: A 2D Vector of transaction IDs (u32), the inner Vecs each represent a block. +/// block_weights: A Vector of total weights per block. +/// clusters: A 2D Vector of transaction IDs representing clusters of dependent mempool transactions +/// rates: A Vector of tuples containing transaction IDs (u32) and effective fee per vsize (f64) #[napi(constructor)] pub struct GbtResult { pub blocks: Vec>, + pub block_weights: Vec, pub clusters: Vec>, pub rates: Vec>, // Tuples not supported. u32 fits inside f64 } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 7da8e0613..757b77c55 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -262,7 +262,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - const processed = this.processBlockTemplates(newMempool, blocks, rates, clusters, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, null, rates, clusters, saveResults); logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -318,7 +318,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - this.processBlockTemplates(newMempool, blocks, rates, clusters, saveResults); + this.processBlockTemplates(newMempool, blocks, null, rates, clusters, saveResults); logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`); } catch (e) { logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); @@ -349,13 +349,13 @@ class MempoolBlocks { // run the block construction algorithm in a separate thread, and wait for a result const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator(); try { - const { blocks, rates, clusters } = this.convertNapiResultTxids( + const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids( await rustGbt.make(new Uint8Array(mempoolBuffer)), ); if (saveResults) { this.rustInitialized = true; } - const processed = this.processBlockTemplates(newMempool, blocks, rates, clusters, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, saveResults); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -395,7 +395,7 @@ class MempoolBlocks { // run the block construction algorithm in a separate thread, and wait for a result try { - const { blocks, rates, clusters } = this.convertNapiResultTxids( + const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids( await this.rustGbtGenerator.update( new Uint8Array(addedBuffer), new Uint8Array(removedBuffer), @@ -406,7 +406,7 @@ class MempoolBlocks { if (expectedMempoolSize !== actualMempoolSize) { throw new Error('GBT returned wrong number of transactions, cache is probably out of sync'); } else { - this.processBlockTemplates(newMempool, blocks, rates, clusters, true); + this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, true); } logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); } catch (e) { @@ -415,10 +415,11 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool, blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }, saveResults): MempoolBlockWithTransactions[] { + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: { [root: string]: number }, clusters: { [root: string]: string[] }, saveResults): MempoolBlockWithTransactions[] { for (const txid of Object.keys(rates)) { if (txid in mempool) { mempool[txid].effectiveFeePerVsize = rates[txid]; + mempool[txid].cpfpChecked = false; } } @@ -426,62 +427,15 @@ class MempoolBlocks { let stackWeight; let feeStatsCalculator: OnlineFeeStatsCalculator | void; if (hasBlockStack) { - stackWeight = blocks[blocks.length - 1].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0); + if (blockWeights && blockWeights[7] !== null) { + stackWeight = blockWeights[7]; + } else { + stackWeight = blocks[blocks.length - 1].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0); + } hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS; feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]); } - const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees, feeStats }[] = []; - const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2; - // update this thread's mempool with the results - for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) { - const block: string[] = blocks[blockIndex]; - let txid: string; - let mempoolTx: MempoolTransactionExtended; - let totalSize = 0; - let totalVsize = 0; - let totalWeight = 0; - let totalFees = 0; - const transactions: MempoolTransactionExtended[] = []; - for (let txIndex = 0; txIndex < block.length; txIndex++) { - txid = block[txIndex]; - if (txid) { - mempoolTx = mempool[txid]; - // save position in projected blocks - mempoolTx.position = { - block: blockIndex, - vsize: totalVsize + (mempoolTx.vsize / 2), - }; - mempoolTx.ancestors = []; - mempoolTx.descendants = []; - mempoolTx.bestDescendant = null; - mempoolTx.cpfpChecked = true; - - // online calculation of stack-of-blocks fee stats - if (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) { - feeStatsCalculator.processNext(mempoolTx); - } - - totalSize += mempoolTx.size; - totalVsize += mempoolTx.vsize; - totalWeight += mempoolTx.weight; - totalFees += mempoolTx.fee; - - if (totalVsize <= sizeLimit) { - transactions.push(mempoolTx); - } - } - } - readyBlocks.push({ - transactionIds: block, - transactions, - totalSize, - totalWeight, - totalFees, - feeStats: (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined, - }); - } - for (const cluster of Object.values(clusters)) { for (const memberTxid of cluster) { if (memberTxid in mempool) { @@ -508,10 +462,66 @@ class MempoolBlocks { mempoolTx.ancestors = ancestors; mempoolTx.descendants = descendants; mempoolTx.bestDescendant = null; + mempoolTx.cpfpChecked = true; } } } + const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees, feeStats }[] = []; + const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2; + // update this thread's mempool with the results + for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) { + const block: string[] = blocks[blockIndex]; + let mempoolTx: MempoolTransactionExtended; + let totalSize = 0; + let totalVsize = 0; + let totalWeight = 0; + let totalFees = 0; + const transactions: MempoolTransactionExtended[] = []; + for (const txid of block) { + if (txid) { + mempoolTx = mempool[txid]; + // save position in projected blocks + mempoolTx.position = { + block: blockIndex, + vsize: totalVsize + (mempoolTx.vsize / 2), + }; + if (!mempoolTx.cpfpChecked) { + if (mempoolTx.ancestors?.length) { + mempoolTx.ancestors = []; + } + if (mempoolTx.descendants?.length) { + mempoolTx.descendants = []; + } + mempoolTx.bestDescendant = null; + mempoolTx.cpfpChecked = true; + } + + // online calculation of stack-of-blocks fee stats + if (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) { + feeStatsCalculator.processNext(mempoolTx); + } + + totalSize += mempoolTx.size; + totalVsize += mempoolTx.vsize; + totalWeight += mempoolTx.weight; + totalFees += mempoolTx.fee; + + if (totalVsize <= sizeLimit) { + transactions.push(mempoolTx); + } + } + } + readyBlocks.push({ + transactionIds: block, + transactions, + totalSize, + totalWeight, + totalFees, + feeStats: (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined, + }); + } + const mempoolBlocks = readyBlocks.map((b) => { return this.dataToMempoolBlocks(b.transactionIds, b.transactions, b.totalSize, b.totalWeight, b.totalFees, b.feeStats); }); @@ -595,8 +605,8 @@ class MempoolBlocks { return { blocks: convertedBlocks, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }}; } - private convertNapiResultTxids({ blocks, rates, clusters }: { blocks: number[][], rates: number[][], clusters: number[][]}) - : { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }} { + private convertNapiResultTxids({ blocks, blockWeights, rates, clusters }: { blocks: number[][], blockWeights: number[], rates: number[][], clusters: number[][]}) + : { blocks: string[][], blockWeights: number[], rates: { [root: string]: number }, clusters: { [root: string]: string[] }} { const rateMap = new Map(); const clusterMap = new Map(); for (const rate of rates) { @@ -634,7 +644,7 @@ class MempoolBlocks { throw new Error('GBT returned a cluster rooted in a transaction with unknown uid'); } } - return { blocks: convertedBlocks, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }}; + return { blocks: convertedBlocks, blockWeights, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], blockWeights: number[], rates: { [root: string]: number }, clusters: { [root: string]: string[] }}; } private mempoolToArrayBuffer(txs: MempoolTransactionExtended[], mempool: { [txid: string]: MempoolTransactionExtended }): ArrayBuffer {