mempool/backend/src/api/mempool-blocks.ts

import logger from '../logger';
import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, TransactionSet, Ancestor } from '../mempool.interfaces';
import { Common } from './common';
import config from '../config';

class MempoolBlocks {
  private mempoolBlocks: MempoolBlockWithTransactions[] = [];
  private mempoolBlockDeltas: MempoolBlockDelta[] = [];

  constructor() {}

  public getMempoolBlocks(): MempoolBlock[] {
    return this.mempoolBlocks.map((block) => {
      return {
        blockSize: block.blockSize,
        blockVSize: block.blockVSize,
        nTx: block.nTx,
        totalFees: block.totalFees,
        medianFee: block.medianFee,
        feeRange: block.feeRange,
      };
    });
  }

  public getMempoolBlocksWithTransactions(): MempoolBlockWithTransactions[] {
    return this.mempoolBlocks;
  }

  public getMempoolBlockDeltas(): MempoolBlockDelta[] {
    return this.mempoolBlockDeltas;
  }

  public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }): void {
    const latestMempool = memPool;
    const memPoolArray: TransactionExtended[] = [];
    for (const i in latestMempool) {
      if (latestMempool.hasOwnProperty(i)) {
        memPoolArray.push(latestMempool[i]);
      }
    }
    const start = new Date().getTime();

    // Clear bestDescendants & ancestors
    memPoolArray.forEach((tx) => {
      tx.bestDescendant = null;
      tx.ancestors = [];
      tx.cpfpChecked = false;
      if (!tx.effectiveFeePerVsize) {
        tx.effectiveFeePerVsize = tx.feePerVsize;
      }
    });

    // First sort
    memPoolArray.sort((a, b) => b.feePerVsize - a.feePerVsize);

    // Loop through and traverse all ancestors and sum up all the sizes + fees
    // Pass down size + fee to all unconfirmed children
    let sizes = 0;
    memPoolArray.forEach((tx, i) => {
      sizes += tx.weight;
      if (sizes > 4000000 * 8) {
        return;
      }
      Common.setRelativesAndGetCpfpInfo(tx, memPool);
    });

    // Final sort, by effective fee
    memPoolArray.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);

    const end = new Date().getTime();
    const time = end - start;
    logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');

    const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
    this.mempoolBlocks = blocks;
    this.mempoolBlockDeltas = deltas;
  }

  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]):
    { blocks: MempoolBlockWithTransactions[], deltas: MempoolBlockDelta[] } {
    const mempoolBlocks: MempoolBlockWithTransactions[] = [];
    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
    let blockWeight = 0;
    let blockSize = 0;
    let transactions: TransactionExtended[] = [];
    transactionsSorted.forEach((tx) => {
      if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
        || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
        blockWeight += tx.weight;
        blockSize += tx.size;
        transactions.push(tx);
      } else {
        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
        blockWeight = tx.weight;
        blockSize = tx.size;
        transactions = [tx];
      }
    });
    if (transactions.length) {
      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
    }

    // Calculate change from previous block states
    for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
      let added: TransactionStripped[] = [];
      let removed: string[] = [];
      if (mempoolBlocks[i] && !prevBlocks[i]) {
        added = mempoolBlocks[i].transactions;
      } else if (!mempoolBlocks[i] && prevBlocks[i]) {
        removed = prevBlocks[i].transactions.map(tx => tx.txid);
      } else if (mempoolBlocks[i] && prevBlocks[i]) {
        const prevIds = {};
        const newIds = {};
        prevBlocks[i].transactions.forEach(tx => {
          prevIds[tx.txid] = true;
        });
        mempoolBlocks[i].transactions.forEach(tx => {
          newIds[tx.txid] = true;
        });
        prevBlocks[i].transactions.forEach(tx => {
          if (!newIds[tx.txid]) {
            removed.push(tx.txid);
          }
        });
        mempoolBlocks[i].transactions.forEach(tx => {
          if (!prevIds[tx.txid]) {
            added.push(tx);
          }
        });
      }
      mempoolBlockDeltas.push({
        added,
        removed
      });
    }

    return {
      blocks: mempoolBlocks,
      deltas: mempoolBlockDeltas
    };
  }

  /*
  * Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core
  * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
  *
  * templateLimit: number of blocks to build using the full algo,
  *     remaining blocks up to blockLimit will skip the expensive updateDescendants step
  *
  * blockLimit: number of blocks to build in total. Excess transactions will be ignored.
  */
  public makeBlockTemplates(mempool: { [txid: string]: TransactionExtended }, templateLimit: number = Infinity, blockLimit: number = Infinity): MempoolBlockWithTransactions[] {
    const start = new Date().getTime();
    const txSets: { [txid: string]: TransactionSet } = {};
    const mempoolArray: TransactionExtended[] = Object.values(mempool);

    mempoolArray.forEach((tx) => {
      tx.bestDescendant = null;
      tx.ancestors = [];
      tx.cpfpChecked = false;
      tx.effectiveFeePerVsize = tx.feePerVsize;
      txSets[tx.txid] = {
        fee: 0,
        weight: 1,
        score: 0,
        children: [],
        available: true,
        modified: false,
      };
    });

    // Build relatives graph & calculate ancestor scores
    mempoolArray.forEach((tx) => {
      this.setRelatives(tx, mempool, txSets);
    });

    // Sort by descending ancestor score
    const byAncestor = (a, b): number => this.sortByAncestorScore(a, b, txSets);
    mempoolArray.sort(byAncestor);

    // Build blocks by greedily choosing the highest feerate package
    // (i.e. the package rooted in the transaction with the best ancestor score)
    const blocks: MempoolBlockWithTransactions[] = [];
    let blockWeight = 4000;
    let blockSize = 0;
    let transactions: TransactionExtended[] = [];
    let modified: TransactionExtended[] = [];
    let overflow: TransactionExtended[] = [];
    let failures = 0;
    while ((mempoolArray.length || modified.length) && blocks.length < blockLimit) {
      const simpleMode = blocks.length >= templateLimit;
      let anyModified = false;
      // Select best next package
      let nextTx;
      if (mempoolArray.length && (!modified.length || txSets[mempoolArray[0].txid]?.score > txSets[modified[0].txid]?.score)) {
        nextTx = mempoolArray.shift();
        if (txSets[nextTx?.txid]?.modified) {
          nextTx = null;
        }
      } else {
        nextTx = modified.shift();
      }

      if (nextTx && txSets[nextTx.txid]?.available) {
        const nextTxSet = txSets[nextTx.txid];
        // Check if the package fits into this block
        if (nextTxSet && blockWeight + nextTxSet.weight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
          blockWeight += nextTxSet.weight;
          // sort txSet by dependency graph (equivalent to sorting by ascending ancestor count)
          const sortedTxSet = [...nextTx.ancestors.sort((a, b) => {
            return (mempool[a.txid]?.ancestors?.length || 0) - (mempool[b.txid]?.ancestors?.length || 0);
          }), nextTx];
          sortedTxSet.forEach((ancestor, i, arr) => {
            const tx = mempool[ancestor.txid];
            const txSet = txSets[ancestor.txid];
            if (txSet.available) {
              txSet.available = false;
              tx.effectiveFeePerVsize = nextTxSet.fee / (nextTxSet.weight / 4);
              tx.cpfpChecked = true;
              if (i < arr.length - 1) {
                tx.bestDescendant = {
                  txid: arr[i + 1].txid,
                  fee: arr[i + 1].fee,
                  weight: arr[i + 1].weight,
                };
              }
              transactions.push(tx);
              blockSize += tx.size;
            }
          });

          // remove these as valid package ancestors for any remaining descendants
          if (!simpleMode) {
            sortedTxSet.forEach(tx => {
              anyModified = this.updateDescendants(tx, tx, mempool, txSets, modified);
            });
          }

          failures = 0;
        } else {
          // hold this package in an overflow list while we check for smaller options
          txSets[nextTx.txid].modified = true;
          overflow.push(nextTx);
          failures++;
        }
      }

      // this block is full
      const outOfTransactions = !mempoolArray.length && !modified.length;
      const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
      const exceededSimpleTries = failures > 0 && simpleMode;
      if (outOfTransactions || exceededPackageTries || exceededSimpleTries) {
        // construct this block
        blocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, blocks.length));
        // reset for the next block
        transactions = [];
        blockSize = 0;
        blockWeight = 4000;

        // 'overflow' packages didn't fit in this block, but are valid candidates for the next
        if (overflow.length) {
          modified = modified.concat(overflow);
          overflow = [];
          anyModified = true;
        }
      }

      // re-sort modified list if necessary
      if (anyModified) {
        modified = modified.filter(tx => txSets[tx.txid]?.available).sort(byAncestor);
      }
    }

    const end = new Date().getTime();
    const time = end - start;
    logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds');

    return blocks;
  }

  private sortByAncestorScore(a, b, txSets): number {
    return txSets[b.txid]?.score - txSets[a.txid]?.score;
  }

  private setRelatives(tx: TransactionExtended, mempool: { [txid: string]: TransactionExtended }, txSets: { [txid: string]: TransactionSet }): { [txid: string]: Ancestor } {
    let ancestors: { [txid: string]: Ancestor } = {};
    tx.vin.forEach((parent) => {
      const parentTx = mempool[parent.txid];
      const parentTxSet = txSets[parent.txid];
      if (parentTx && parentTxSet) {
        ancestors[parentTx.txid] = parentTx;
        if (!parentTxSet.children) {
          parentTxSet.children = [tx.txid];
        } else {
          parentTxSet.children.push(tx.txid);
        }
        if (!parentTxSet.score) {
          ancestors = {
            ...ancestors,
            ...this.setRelatives(parentTx, mempool, txSets),
          };
        }
      }
    });
    tx.ancestors = Object.values(ancestors).map(ancestor => {
      return {
        txid: ancestor.txid,
        fee: ancestor.fee,
        weight: ancestor.weight
      };
    });
    let totalFees = tx.fee;
    let totalWeight = tx.weight;
    tx.ancestors.forEach(ancestor => {
      totalFees += ancestor.fee;
      totalWeight += ancestor.weight;
    });
    txSets[tx.txid].fee = totalFees;
    txSets[tx.txid].weight = totalWeight;
    txSets[tx.txid].score = this.calcAncestorScore(tx, totalFees, totalWeight);

    return ancestors;
  }

  private calcAncestorScore(tx: TransactionExtended, ancestorFees: number, ancestorWeight: number): number {
    return Math.min(tx.fee / tx.weight, ancestorFees / ancestorWeight);
  }

  // walk over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
  // returns whether any descendants were modified
  private updateDescendants(
    root: TransactionExtended,
    tx: TransactionExtended,
    mempool: { [txid: string]: TransactionExtended },
    txSets: { [txid: string]: TransactionSet },
    modified: TransactionExtended[],
  ): boolean {
    let anyModified = false;
    const txSet = txSets[tx.txid];
    if (txSet.children) {
      txSet.children.forEach(childId => {
        const child = mempool[childId];
        if (child && child.ancestors) {
          const ancestorIndex = child.ancestors.findIndex(a => a.txid === root.txid);
          if (ancestorIndex > -1) {
            // remove tx as ancestor
            child.ancestors.splice(ancestorIndex, 1);
            const childTxSet = txSets[childId];
            childTxSet.fee -= root.fee;
            childTxSet.weight -= root.weight;
            childTxSet.score = this.calcAncestorScore(child, childTxSet.fee, childTxSet.weight);
            anyModified = true;

            if (!childTxSet.modified) {
              childTxSet.modified = true;
              modified.push(child);
            }
          }
        }
        // recursively update grandchildren
        if (child) {
          anyModified = this.updateDescendants(root, child, mempool, txSets, modified) || anyModified;
        }
      });
    }
    return anyModified;
  }

  private dataToMempoolBlocks(transactions: TransactionExtended[],
    blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions {
    let rangeLength = 4;
    if (blocksIndex === 0) {
      rangeLength = 8;
    }
    if (transactions.length > 4000) {
      rangeLength = 6;
    } else if (transactions.length > 10000) {
      rangeLength = 8;
    }
    return {
      blockSize: blockSize,
      blockVSize: blockWeight / 4,
      nTx: transactions.length,
      totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
      medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
      feeRange: Common.getFeesInRange(transactions, rangeLength),
      transactionIds: transactions.map((tx) => tx.txid),
      transactions: transactions.map((tx) => Common.stripTransaction(tx)),
    };
  }
}

export default new MempoolBlocks();