New projected block transaction selection algo

backend/src/api/mempool-blocks.ts

@@ -1,5 +1,5 @@
 import logger from '../logger';
-import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces';
+import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, TransactionSet, Ancestor } from '../mempool.interfaces';
 import { Common } from './common';
 import config from '../config';
 
@@ -99,6 +99,7 @@ class MempoolBlocks {
     if (transactions.length) {
       mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
     }
+
     // Calculate change from previous block states
     for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
       let added: TransactionStripped[] = [];
@@ -132,12 +133,238 @@ class MempoolBlocks {
         removed
       });
     }
+
     return {
       blocks: mempoolBlocks,
       deltas: mempoolBlockDeltas
     };
   }
 
+  /*
+  * Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core
+  * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
+  *
+  * templateLimit: number of blocks to build using the full algo,
+  *     remaining blocks up to blockLimit will skip the expensive updateDescendants step
+  *
+  * blockLimit: number of blocks to build in total. Excess transactions will be ignored.
+  */
+  public makeBlockTemplates(mempool: { [txid: string]: TransactionExtended }, templateLimit: number = Infinity, blockLimit: number = Infinity): MempoolBlockWithTransactions[] {
+    const start = new Date().getTime();
+    const txSets: { [txid: string]: TransactionSet } = {};
+    const mempoolArray: TransactionExtended[] = Object.values(mempool);
+
+    mempoolArray.forEach((tx) => {
+      tx.bestDescendant = null;
+      tx.ancestors = [];
+      tx.cpfpChecked = false;
+      tx.effectiveFeePerVsize = tx.feePerVsize;
+      txSets[tx.txid] = {
+        fee: 0,
+        weight: 1,
+        score: 0,
+        children: [],
+        available: true,
+        modified: false,
+      };
+    });
+
+    // Build relatives graph & calculate ancestor scores
+    mempoolArray.forEach((tx) => {
+      this.setRelatives(tx, mempool, txSets);
+    });
+
+    // Sort by descending ancestor score
+    const byAncestor = (a, b): number => this.sortByAncestorScore(a, b, txSets);
+    mempoolArray.sort(byAncestor);
+
+    // Build blocks by greedily choosing the highest feerate package
+    // (i.e. the package rooted in the transaction with the best ancestor score)
+    const blocks: MempoolBlockWithTransactions[] = [];
+    let blockWeight = 4000;
+    let blockSize = 0;
+    let transactions: TransactionExtended[] = [];
+    let modified: TransactionExtended[] = [];
+    let overflow: TransactionExtended[] = [];
+    let failures = 0;
+    while ((mempoolArray.length || modified.length) && blocks.length < blockLimit) {
+      const simpleMode = blocks.length >= templateLimit;
+      let anyModified = false;
+      // Select best next package
+      let nextTx;
+      if (mempoolArray.length && (!modified.length || txSets[mempoolArray[0].txid]?.score > txSets[modified[0].txid]?.score)) {
+        nextTx = mempoolArray.shift();
+        if (txSets[nextTx?.txid]?.modified) {
+          nextTx = null;
+        }
+      } else {
+        nextTx = modified.shift();
+      }
+
+      if (nextTx && txSets[nextTx.txid]?.available) {
+        const nextTxSet = txSets[nextTx.txid];
+        // Check if the package fits into this block
+        if (nextTxSet && blockWeight + nextTxSet.weight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+          blockWeight += nextTxSet.weight;
+          // sort txSet by dependency graph (equivalent to sorting by ascending ancestor count)
+          const sortedTxSet = nextTx.ancestors.sort((a, b) => {
+            return (mempool[a.txid]?.ancestors?.length || 0) - (mempool[b.txid]?.ancestors?.length || 0);
+          });
+          [...sortedTxSet, nextTx].forEach((ancestor, i, arr) => {
+            const tx = mempool[ancestor.txid];
+            const txSet = txSets[ancestor.txid];
+            if (txSet.available) {
+              txSet.available = false;
+              tx.effectiveFeePerVsize = nextTxSet.fee / (nextTxSet.weight / 4);
+              tx.cpfpChecked = true;
+              if (i < arr.length - 1) {
+                tx.bestDescendant = {
+                  txid: arr[i + 1].txid,
+                  fee: arr[i + 1].fee,
+                  weight: arr[i + 1].weight,
+                };
+              }
+              transactions.push(tx);
+              blockSize += tx.size;
+            }
+          });
+
+          // remove these as valid package ancestors for any remaining descendants
+          if (!simpleMode) {
+            sortedTxSet.forEach(tx => {
+              anyModified = this.updateDescendants(tx, tx, mempool, txSets, modified);
+            });
+          }
+
+          failures = 0;
+        } else {
+          // hold this package in an overflow list while we check for smaller options
+          txSets[nextTx.txid].modified = true;
+          overflow.push(nextTx);
+          failures++;
+        }
+      }
+
+      // this block is full
+      const outOfTransactions = !mempoolArray.length && !modified.length;
+      const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
+      const exceededSimpleTries = failures > 0 && simpleMode;
+      if (outOfTransactions || exceededPackageTries || exceededSimpleTries) {
+        // construct this block
+        blocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, blocks.length));
+        // reset for the next block
+        transactions = [];
+        blockSize = 0;
+        blockWeight = 4000;
+
+        // 'overflow' packages didn't fit in this block, but are valid candidates for the next
+        if (overflow.length) {
+          modified = modified.concat(overflow);
+          overflow = [];
+          anyModified = true;
+        }
+      }
+
+      // re-sort modified list if necessary
+      if (anyModified) {
+        modified = modified.filter(tx => txSets[tx.txid]?.available).sort(byAncestor);
+      }
+    }
+
+    const end = new Date().getTime();
+    const time = end - start;
+    logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds');
+
+    return blocks;
+  }
+
+  private sortByAncestorScore(a, b, txSets): number {
+    return txSets[b.txid]?.score - txSets[a.txid]?.score;
+  }
+
+  private setRelatives(tx: TransactionExtended, mempool: { [txid: string]: TransactionExtended }, txSets: { [txid: string]: TransactionSet }): { [txid: string]: Ancestor } {
+    let ancestors: { [txid: string]: Ancestor } = {};
+    tx.vin.forEach((parent) => {
+      const parentTx = mempool[parent.txid];
+      const parentTxSet = txSets[parent.txid];
+      if (parentTx && parentTxSet) {
+        ancestors[parentTx.txid] = parentTx;
+        if (!parentTxSet.children) {
+          parentTxSet.children = [tx.txid];
+        } else {
+          parentTxSet.children.push(tx.txid);
+        }
+        if (!parentTxSet.score) {
+          ancestors = {
+            ...ancestors,
+            ...this.setRelatives(parentTx, mempool, txSets),
+          };
+        }
+      }
+    });
+    tx.ancestors = Object.values(ancestors).map(ancestor => {
+      return {
+        txid: ancestor.txid,
+        fee: ancestor.fee,
+        weight: ancestor.weight
+      };
+    });
+    let totalFees = tx.fee;
+    let totalWeight = tx.weight;
+    tx.ancestors.forEach(ancestor => {
+      totalFees += ancestor.fee;
+      totalWeight += ancestor.weight;
+    });
+    txSets[tx.txid].fee = totalFees;
+    txSets[tx.txid].weight = totalWeight;
+    txSets[tx.txid].score = this.calcAncestorScore(tx, totalFees, totalWeight);
+
+    return ancestors;
+  }
+
+  private calcAncestorScore(tx: TransactionExtended, ancestorFees: number, ancestorWeight: number): number {
+    return Math.min(tx.fee / tx.weight, ancestorFees / ancestorWeight);
+  }
+
+  // walk over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
+  // returns whether any descendants were modified
+  private updateDescendants(
+    root: TransactionExtended,
+    tx: TransactionExtended,
+    mempool: { [txid: string]: TransactionExtended },
+    txSets: { [txid: string]: TransactionSet },
+    modified: TransactionExtended[],
+  ): boolean {
+    let anyModified = false;
+    const txSet = txSets[tx.txid];
+    if (txSet.children) {
+      txSet.children.forEach(childId => {
+        const child = mempool[childId];
+        if (child && child.ancestors && txSets[childId]?.available) {
+          const ancestorIndex = child.ancestors.findIndex(a => a.txid === root.txid);
+          if (ancestorIndex > -1) {
+            // remove tx as ancestor
+            child.ancestors.splice(ancestorIndex, 1);
+            const childTxSet = txSets[childId];
+            childTxSet.fee -= root.fee;
+            childTxSet.weight -= root.weight;
+            childTxSet.score = this.calcAncestorScore(child, childTxSet.fee, childTxSet.weight);
+            anyModified = true;
+
+            if (!childTxSet.modified) {
+              childTxSet.modified = true;
+              modified.push(child);
+            }
+
+            // recursively update grandchildren
+            anyModified = this.updateDescendants(root, child, mempool, txSets, modified) || anyModified;
+          }
+        }
+      });
+    }
+    return anyModified;
+  }
+
   private dataToMempoolBlocks(transactions: TransactionExtended[],
     blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions {
     let rangeLength = 4;

backend/src/api/websocket-handler.ts

@@ -414,15 +414,15 @@ class WebsocketHandler {
     let mBlockDeltas: undefined | MempoolBlockDelta[];
     let matchRate = 0;
     const _memPool = memPool.getMempool();
-    const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+    const projectedBlocks = mempoolBlocks.makeBlockTemplates(cloneMempool(_memPool), 1, 1);
 
-    if (_mempoolBlocks[0]) {
+    if (projectedBlocks[0]) {
       const matches: string[] = [];
       const added: string[] = [];
       const missing: string[] = [];
 
       for (const txId of txIds) {
-        if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
+        if (projectedBlocks[0].transactionIds.indexOf(txId) > -1) {
           matches.push(txId);
         } else {
           added.push(txId);
@@ -430,7 +430,7 @@ class WebsocketHandler {
         delete _memPool[txId];
       }
 
-      for (const txId of _mempoolBlocks[0].transactionIds) {
+      for (const txId of projectedBlocks[0].transactionIds) {
         if (matches.includes(txId) || added.includes(txId)) {
           continue;
         }
@@ -443,14 +443,14 @@ class WebsocketHandler {
       mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
 
       if (Common.indexingEnabled()) {
-        const stripped = _mempoolBlocks[0].transactions.map((tx) => {
+        const stripped = projectedBlocks[0].transactions.map((tx) => {
           return {
             txid: tx.txid,
             vsize: tx.vsize,
             fee: tx.fee ? Math.round(tx.fee) : 0,
             value: tx.value,
           };
-        });  
+        });
         BlocksSummariesRepository.$saveSummary({
           height: block.height,
           template: {
@@ -580,4 +580,14 @@ class WebsocketHandler {
   }
 }
 
+function cloneMempool(mempool: { [txid: string]: TransactionExtended }): { [txid: string]: TransactionExtended } {
+  const cloned = {};
+  Object.keys(mempool).forEach(id => {
+    cloned[id] = {
+      ...mempool[id]
+    };
+  });
+  return cloned;
+}
+
 export default new WebsocketHandler();

backend/src/mempool.interfaces.ts

@@ -70,12 +70,21 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   deleteAfter?: number;
 }
 
-interface Ancestor {
+export interface Ancestor {
   txid: string;
   weight: number;
   fee: number;
 }
 
+export interface TransactionSet {
+  fee: number;
+  weight: number;
+  score: number;
+  children?: string[];
+  available?: boolean;
+  modified?: boolean;
+}
+
 interface BestDescendant {
   txid: string;
   weight: number;