From 130bdac58cc7a833bfbac5ecff00b54355c67891 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 27 Feb 2024 18:09:15 +0000 Subject: [PATCH] Calculate & save acceleration bid boost rates --- backend/src/api/acceleration.ts | 738 ++++++++++++++++++ backend/src/api/database-migration.ts | 24 +- backend/src/api/websocket-handler.ts | 25 +- .../repositories/AccelerationRepository.ts | 43 + 4 files changed, 828 insertions(+), 2 deletions(-) create mode 100644 backend/src/api/acceleration.ts create mode 100644 backend/src/repositories/AccelerationRepository.ts diff --git a/backend/src/api/acceleration.ts b/backend/src/api/acceleration.ts new file mode 100644 index 000000000..2b032d09f --- /dev/null +++ b/backend/src/api/acceleration.ts @@ -0,0 +1,738 @@ +import logger from '../logger'; +import { MempoolTransactionExtended } from '../mempool.interfaces'; +import { IEsploraApi } from './bitcoin/esplora-api.interface'; + +const BLOCK_WEIGHT_UNITS = 4_000_000; +const BLOCK_SIGOPS = 80_000; +const MAX_RELATIVE_GRAPH_SIZE = 200; +const BID_BOOST_WINDOW = 40_000; +const BID_BOOST_MIN_OFFSET = 10_000; +const BID_BOOST_MAX_OFFSET = 400_000; + +type Acceleration = { + txid: string; + max_bid: number; +}; + +interface TxSummary { + txid: string; // txid of the current transaction + effectiveVsize: number; // Total vsize of the dependency tree + effectiveFee: number; // Total fee of the dependency tree in sats + ancestorCount: number; // Number of ancestors +} + +export interface AccelerationInfo { + txSummary: TxSummary; + targetFeeRate: number; // target fee rate (recommended next block fee, or median fee for mined block) + nextBlockFee: number; // fee in sats required to be in the next block (using recommended next block fee, or median fee for mined block) + cost: number; // additional cost to accelerate ((cost + txSummary.effectiveFee) / txSummary.effectiveVsize) >= targetFeeRate +} + +interface GraphTx { + txid: string; + vsize: number; + weight: number; + fees: { + base: number; + }; + depends: string[]; + spentby: string[]; +} + +interface MempoolTx extends GraphTx { + ancestorcount: number; + ancestorsize: number; + fees: { + base: number; + ancestor: number; + }; + + ancestors: Map, + ancestorRate: number; + individualRate: number; + score: number; +} + +class AccelerationCosts { + /** + * Takes a list of accelerations and verbose block data + * Returns the "fair" boost rate to charge accelerations + * + * @param accelerationsx + * @param verboseBlock + */ + public calculateBoostRate(accelerations: Acceleration[], blockTxs: IEsploraApi.Transaction[]): number { + // Run GBT ourselves to calculate accurate effective fee rates + // the list of transactions comes from a mined block, so we already know everything fits within consensus limits + const template = makeBlockTemplate(blockTxs, accelerations, 1, Infinity, Infinity); + + // initialize working maps for fast tx lookups + const accMap = {}; + const txMap = {}; + for (const acceleration of accelerations) { + accMap[acceleration.txid] = acceleration; + } + for (const tx of template) { + txMap[tx.txid] = tx; + } + + // Identify and exclude accelerated and otherwise prioritized transactions + const excludeMap = {}; + let totalWeight = 0; + let minAcceleratedPackage = Infinity; + let lastEffectiveRate = 0; + // Iterate over the mined template from bottom to top. + // Transactions should appear in ascending order of mining priority. + for (const blockTx of [...blockTxs].reverse()) { + const txid = blockTx.txid; + const tx = txMap[txid]; + totalWeight += tx.weight; + const isAccelerated = accMap[txid] != null; + // If a cluster has a in-band effective fee rate than the previous cluster, + // it must have been prioritized out-of-band (in order to have a higher mining priority) + // so exclude from the analysis. + const isPrioritized = tx.effectiveFeePerVsize < lastEffectiveRate; + if (isPrioritized || isAccelerated) { + let packageWeight = 0; + // exclude this whole CPFP cluster + for (const clusterTxid of tx.cluster) { + packageWeight += txMap[clusterTxid].weight; + if (!excludeMap[clusterTxid]) { + excludeMap[clusterTxid] = true; + } + } + // keep track of the smallest accelerated CPFP cluster for later + if (isAccelerated) { + minAcceleratedPackage = Math.min(minAcceleratedPackage, packageWeight); + } + } + if (!isPrioritized) { + if (!isAccelerated || !lastEffectiveRate) { + lastEffectiveRate = tx.effectiveFeePerVsize; + } + } + } + + // The Bid Boost Rate is calculated by disregarding the bottom X weight units of the block, + // where X is the larger of BID_BOOST_MIN_OFFSET or the smallest accelerated package weight (the "offset"), + // then taking the average fee rate of the following BID_BOOST_WINDOW weight units + // (ignoring accelerated transactions and their ancestors). + // + // Transactions within the offset might pay less than the fair rate due to bin-packing effects + // But the average rate paid by the next chunk of non-accelerated transactions provides a good + // upper bound on the "next best rate" of alternatives to including the accelerated transactions + // (since, if there were any better options, they would have been included instead) + const spareWeight = BLOCK_WEIGHT_UNITS - totalWeight; + const windowOffset = Math.min(Math.max(minAcceleratedPackage, BID_BOOST_MIN_OFFSET, spareWeight), BID_BOOST_MAX_OFFSET); + const leftBound = windowOffset; + const rightBound = windowOffset + BID_BOOST_WINDOW; + let totalFeeInWindow = 0; + let totalWeightInWindow = Math.max(0, spareWeight - leftBound); + let txIndex = blockTxs.length - 1; + for (let offset = spareWeight; offset < BLOCK_WEIGHT_UNITS && txIndex >= 0; txIndex--) { + const txid = blockTxs[txIndex].txid; + const tx = txMap[txid]; + if (excludeMap[txid]) { + // skip prioritized transactions and their ancestors + continue; + } + + const left = offset; + const right = offset + tx.weight; + offset += tx.weight; + if (right < leftBound) { + // not within window yet + continue; + } + if (left > rightBound) { + // past window + break; + } + // count fees for weight units within the window + const overlapLeft = Math.max(leftBound, left); + const overlapRight = Math.min(rightBound, right); + const overlapUnits = overlapRight - overlapLeft; + totalFeeInWindow += (tx.effectiveFeePerVsize * (overlapUnits / 4)); + totalWeightInWindow += overlapUnits; + } + + if (totalWeightInWindow < BID_BOOST_WINDOW) { + // not enough un-prioritized transactions to calculate a fair rate + // just charge everyone their max bids + return Infinity; + } + // Divide the total fee by the size of the BID_BOOST_WINDOW in vbytes + const averageRate = totalFeeInWindow / (BID_BOOST_WINDOW / 4); + return averageRate; + } + + + /** + * Takes an accelerated mined txid and a target rate + * Returns the total vsize, fees and acceleration cost (in sats) of the tx and all same-block ancestors + * + * @param txid + * @param medianFeeRate + */ + public getAccelerationInfo(tx: MempoolTransactionExtended, targetFeeRate: number, transactions: MempoolTransactionExtended[]): AccelerationInfo { + // Get same-block transaction ancestors + const allRelatives = this.getSameBlockRelatives(tx, transactions); + const relativesMap = this.initializeRelatives(allRelatives); + const rootTx = relativesMap.get(tx.txid) as MempoolTx; + + // Calculate cost to boost + return this.calculateAccelerationAncestors(rootTx, relativesMap, targetFeeRate); + } + + /** + * Takes a raw transaction, and builds a graph of same-block relatives, + * and returns as a MempoolTx + * + * @param tx + */ + private getSameBlockRelatives(tx: MempoolTransactionExtended, transactions: MempoolTransactionExtended[]): Map { + const blockTxs = new Map(); // map of txs in this block + const spendMap = new Map(); // map of outpoints to spending txids + for (const tx of transactions) { + blockTxs.set(tx.txid, tx); + for (const vin of tx.vin) { + spendMap.set(`${vin.txid}:${vin.vout}`, tx.txid); + } + } + + const relatives: Map = new Map(); + const stack: string[] = [tx.txid]; + + // build set of same-block ancestors + while (stack.length > 0) { + const nextTxid = stack.pop(); + const nextTx = nextTxid ? blockTxs.get(nextTxid) : null; + if (!nextTx || relatives.has(nextTx.txid)) { + continue; + } + + const mempoolTx = this.convertToGraphTx(nextTx); + + mempoolTx.fees.base = nextTx.fee || 0; + mempoolTx.depends = nextTx.vin.map(vin => vin.txid).filter(inTxid => inTxid && blockTxs.has(inTxid)) as string[]; + mempoolTx.spentby = nextTx.vout.map((vout, index) => spendMap.get(`${nextTx.txid}:${index}`)).filter(outTxid => outTxid && blockTxs.has(outTxid)) as string[]; + + for (const txid of [...mempoolTx.depends, ...mempoolTx.spentby]) { + if (txid) { + stack.push(txid); + } + } + + relatives.set(mempoolTx.txid, mempoolTx); + } + + return relatives; + } + + /** + * Takes a raw transaction and converts it to MempoolTx format + * fee and ancestor data is initialized with dummy/null values + * + * @param tx + */ + private convertToGraphTx(tx: MempoolTransactionExtended): GraphTx { + return { + txid: tx.txid, + vsize: tx.vsize, + weight: tx.weight, + fees: { + base: 0, // dummy + }, + depends: [], // dummy + spentby: [], //dummy + }; + } + + private convertGraphToMempoolTx(tx: GraphTx): MempoolTx { + return { + ...tx, + fees: { + base: tx.fees.base, + ancestor: tx.fees.base, + }, + ancestorcount: 1, + ancestorsize: tx.vsize, + ancestors: new Map(), + ancestorRate: 0, + individualRate: 0, + score: 0, + }; + } + + /** + * Given a root transaction, a list of in-mempool ancestors, and a target fee rate, + * Calculate the minimum set of transactions to fee-bump, their total vsize + fees + * + * @param tx + * @param ancestors + */ + private calculateAccelerationAncestors(tx: MempoolTx, relatives: Map, targetFeeRate: number): AccelerationInfo { + // add root tx to the ancestor map + relatives.set(tx.txid, tx); + + // Check for high-sigop transactions (not supported) + relatives.forEach(entry => { + if (entry.vsize > Math.ceil(entry.weight / 4)) { + throw new Error(`high_sigop_tx`); + } + }); + + // Initialize individual & ancestor fee rates + relatives.forEach(entry => this.setAncestorScores(entry)); + + // Sort by descending ancestor score + let sortedRelatives = Array.from(relatives.values()).sort(this.mempoolComparator); + + let includedInCluster: Map | null = null; + + // While highest score >= targetFeeRate + let maxIterations = MAX_RELATIVE_GRAPH_SIZE; + while (sortedRelatives.length && sortedRelatives[0].score && sortedRelatives[0].score >= targetFeeRate && maxIterations > 0) { + maxIterations--; + // Grab the highest scoring entry + const best = sortedRelatives.shift(); + if (best) { + const cluster = new Map(best.ancestors?.entries() || []); + if (best.ancestors.has(tx.txid)) { + includedInCluster = cluster; + } + cluster.set(best.txid, best); + // Remove this cluster (it already pays over the target rate, so doesn't need to be boosted) + // and update scores, ancestor totals and dependencies for the survivors + this.removeAncestors(cluster, relatives); + + // re-sort + sortedRelatives = Array.from(relatives.values()).sort(this.mempoolComparator); + } + } + + // sanity check for infinite loops / too many ancestors (should never happen) + if (maxIterations <= 0) { + logger.warn(`acceleration dependency calculation failed: calculateAccelerationAncestors loop exceeded ${MAX_RELATIVE_GRAPH_SIZE} iterations, unable to proceed`); + throw new Error('invalid_tx_dependencies'); + } + + let totalFee = Math.round(tx.fees.ancestor * 100_000_000); + + // transaction is already CPFP-d above the target rate by some descendant + if (includedInCluster) { + let clusterSize = 0; + let clusterFee = 0; + includedInCluster.forEach(entry => { + clusterSize += entry.vsize; + clusterFee += (entry.fees.base * 100_000_000); + }); + const clusterRate = clusterFee / clusterSize; + totalFee = Math.ceil(tx.ancestorsize * clusterRate); + } + + // Whatever remains in the accelerated tx's dependencies needs to be boosted to the targetFeeRate + // Cost = (totalVsize * targetFeeRate) - totalFee + return { + txSummary: { + txid: tx.txid, + effectiveVsize: tx.ancestorsize, + effectiveFee: totalFee, + ancestorCount: tx.ancestorcount, + }, + cost: Math.max(0, Math.ceil(tx.ancestorsize * targetFeeRate) - totalFee), + targetFeeRate, + nextBlockFee: Math.ceil(tx.ancestorsize * targetFeeRate), + }; + } + + /** + * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors + * for each transaction. + * + * @param tx + * @param all + */ + private setAncestors(tx: MempoolTx, all: Map, visited: Map>, depth: number = 0): Map { + // sanity check for infinite recursion / too many ancestors (should never happen) + if (depth >= 100) { + logger.warn('acceleration dependency calculation failed: setAncestors reached depth of 100, unable to proceed', `Accelerator`); + throw new Error('invalid_tx_dependencies'); + } + + // initialize the ancestor map for this tx + tx.ancestors = new Map(); + tx.depends.forEach(parentId => { + const parent = all.get(parentId); + if (parent) { + // add the parent + tx.ancestors?.set(parentId, parent); + // check for a cached copy of this parent's ancestors + let ancestors = visited.get(parent.txid); + if (!ancestors) { + // recursively fetch the parent's ancestors + ancestors = this.setAncestors(parent, all, visited, depth + 1); + } + // and add to this tx's map + ancestors.forEach((ancestor, ancestorId) => { + tx.ancestors?.set(ancestorId, ancestor); + }); + } + }); + visited.set(tx.txid, tx.ancestors); + + return tx.ancestors; + } + + /** + * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph + * by running setAncestors on each leaf, and caching intermediate results. + * then initializes ancestor data for each transaction + * + * @param all + */ + private initializeRelatives(all: Map): Map { + const mempoolTxs = new Map(); + all.forEach(entry => { + mempoolTxs.set(entry.txid, this.convertGraphToMempoolTx(entry)); + }); + const visited: Map> = new Map(); + const leaves: MempoolTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0); + for (const leaf of leaves) { + this.setAncestors(leaf, mempoolTxs, visited); + } + mempoolTxs.forEach(entry => { + entry.ancestors?.forEach(ancestor => { + entry.ancestorcount++; + entry.ancestorsize += ancestor.vsize; + entry.fees.ancestor += ancestor.fees.base; + }); + this.setAncestorScores(entry); + }); + return mempoolTxs; + } + + /** + * Remove a cluster of transactions from an in-mempool dependency graph + * and update the survivors' scores and ancestors + * + * @param cluster + * @param ancestors + */ + private removeAncestors(cluster: Map, all: Map): void { + // remove + cluster.forEach(tx => { + all.delete(tx.txid); + }); + + // update survivors + all.forEach(tx => { + cluster.forEach(remove => { + if (tx.ancestors?.has(remove.txid)) { + // remove as dependency + tx.ancestors.delete(remove.txid); + tx.depends = tx.depends.filter(parent => parent !== remove.txid); + // update ancestor sizes and fees + tx.ancestorsize -= remove.vsize; + tx.fees.ancestor -= remove.fees.base; + } + }); + // recalculate fee rates + this.setAncestorScores(tx); + }); + } + + /** + * Take a mempool transaction, and set the fee rates and ancestor score + * + * @param tx + */ + private setAncestorScores(tx: MempoolTx): void { + tx.individualRate = (tx.fees.base * 100_000_000) / tx.vsize; + tx.ancestorRate = (tx.fees.ancestor * 100_000_000) / tx.ancestorsize; + tx.score = Math.min(tx.individualRate, tx.ancestorRate); + } + + // Sort by descending score + private mempoolComparator(a, b): number { + return b.score - a.score; + } +} + +export default new AccelerationCosts; + +interface TemplateTransaction { + txid: string; + order: number; + weight: number; + adjustedVsize: number; // sigop-adjusted vsize, rounded up to the nearest integer + sigops: number; + fee: number; + feeDelta: number; + ancestors: string[]; + cluster: string[]; + effectiveFeePerVsize: number; +} + +interface MinerTransaction extends TemplateTransaction { + inputs: string[]; + feePerVsize: number; + relativesSet: boolean; + ancestorMap: Map; + children: Set; + ancestorFee: number; + ancestorVsize: number; + ancestorSigops: number; + score: number; + used: boolean; + modified: boolean; + dependencyRate: number; +} + +/* +* Build a block using an approximation of the transaction selection algorithm from Bitcoin Core +* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) +*/ +function makeBlockTemplate(candidates: IEsploraApi.Transaction[], accelerations: Acceleration[], maxBlocks: number = 8, weightLimit: number = BLOCK_WEIGHT_UNITS, sigopLimit: number = BLOCK_SIGOPS): TemplateTransaction[] { + const auditPool: Map = new Map(); + const mempoolArray: MinerTransaction[] = []; + + candidates.forEach(tx => { + // initializing everything up front helps V8 optimize property access later + const adjustedVsize = Math.ceil(Math.max(tx.weight / 4, 5 * (tx.sigops || 0))); + const feePerVsize = (tx.fee / adjustedVsize); + auditPool.set(tx.txid, { + txid: tx.txid, + order: txidToOrdering(tx.txid), + fee: tx.fee, + feeDelta: 0, + weight: tx.weight, + adjustedVsize, + feePerVsize: feePerVsize, + effectiveFeePerVsize: feePerVsize, + dependencyRate: feePerVsize, + sigops: tx.sigops || 0, + inputs: (tx.vin?.map(vin => vin.txid) || []) as string[], + relativesSet: false, + ancestors: [], + cluster: [], + ancestorMap: new Map(), + children: new Set(), + ancestorFee: 0, + ancestorVsize: 0, + ancestorSigops: 0, + score: 0, + used: false, + modified: false, + }); + mempoolArray.push(auditPool.get(tx.txid) as MinerTransaction); + }); + + // set accelerated effective fee + for (const acceleration of accelerations) { + const tx = auditPool.get(acceleration.txid); + if (tx) { + tx.feeDelta = acceleration.max_bid; + tx.feePerVsize = ((tx.fee + tx.feeDelta) / tx.adjustedVsize); + tx.effectiveFeePerVsize = tx.feePerVsize; + tx.dependencyRate = tx.feePerVsize; + } + } + + // Build relatives graph & calculate ancestor scores + for (const tx of mempoolArray) { + if (!tx.relativesSet) { + setRelatives(tx, auditPool); + } + } + + // Sort by descending ancestor score + mempoolArray.sort(priorityComparator); + + // Build blocks by greedily choosing the highest feerate package + // (i.e. the package rooted in the transaction with the best ancestor score) + const blocks: number[][] = []; + let blockWeight = 0; + let blockSigops = 0; + const transactions: MinerTransaction[] = []; + let modified: MinerTransaction[] = []; + const overflow: MinerTransaction[] = []; + let failures = 0; + while (mempoolArray.length || modified.length) { + // skip invalid transactions + while (mempoolArray[0].used || mempoolArray[0].modified) { + mempoolArray.shift(); + } + + // Select best next package + let nextTx; + const nextPoolTx = mempoolArray[0]; + const nextModifiedTx = modified[0]; + if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) { + nextTx = nextPoolTx; + mempoolArray.shift(); + } else { + modified.shift(); + if (nextModifiedTx) { + nextTx = nextModifiedTx; + } + } + + if (nextTx && !nextTx?.used) { + // Check if the package fits into this block + if (blocks.length >= (maxBlocks - 1) || ((blockWeight + (4 * nextTx.ancestorVsize) < weightLimit) && (blockSigops + nextTx.ancestorSigops <= sigopLimit))) { + const ancestors: MinerTransaction[] = Array.from(nextTx.ancestorMap.values()); + // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count) + const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx]; + const clusterTxids = sortedTxSet.map(tx => tx.txid); + const effectiveFeeRate = Math.min(nextTx.dependencyRate || Infinity, nextTx.ancestorFee / nextTx.ancestorVsize); + const used: MinerTransaction[] = []; + while (sortedTxSet.length) { + const ancestor = sortedTxSet.pop(); + if (!ancestor) { + continue; + } + ancestor.used = true; + ancestor.usedBy = nextTx.txid; + // update this tx with effective fee rate & relatives data + if (ancestor.effectiveFeePerVsize !== effectiveFeeRate) { + ancestor.effectiveFeePerVsize = effectiveFeeRate; + } + ancestor.cluster = clusterTxids; + transactions.push(ancestor); + blockWeight += ancestor.weight; + blockSigops += ancestor.sigops; + used.push(ancestor); + } + + // remove these as valid package ancestors for any descendants remaining in the mempool + if (used.length) { + used.forEach(tx => { + modified = updateDescendants(tx, auditPool, modified, effectiveFeeRate); + }); + } + + failures = 0; + } else { + // hold this package in an overflow list while we check for smaller options + overflow.push(nextTx); + failures++; + } + } + + // this block is full + const exceededPackageTries = failures > 1000 && blockWeight > (weightLimit - 4000); + const queueEmpty = !mempoolArray.length && !modified.length; + + if (exceededPackageTries || queueEmpty) { + break; + } + } + + for (const tx of transactions) { + tx.ancestors = Object.values(tx.ancestorMap); + } + + return transactions; +} + +// traverse in-mempool ancestors +// recursion unavoidable, but should be limited to depth < 25 by mempool policy +function setRelatives( + tx: MinerTransaction, + mempool: Map, +): void { + for (const parent of tx.inputs) { + const parentTx = mempool.get(parent); + if (parentTx && !tx.ancestorMap?.has(parent)) { + tx.ancestorMap.set(parent, parentTx); + parentTx.children.add(tx); + // visit each node only once + if (!parentTx.relativesSet) { + setRelatives(parentTx, mempool); + } + parentTx.ancestorMap.forEach((ancestor) => { + tx.ancestorMap.set(ancestor.txid, ancestor); + }); + } + }; + tx.ancestorFee = (tx.fee + tx.feeDelta); + tx.ancestorVsize = tx.adjustedVsize || 0; + tx.ancestorSigops = tx.sigops || 0; + tx.ancestorMap.forEach((ancestor) => { + tx.ancestorFee += (ancestor.fee + ancestor.feeDelta); + tx.ancestorVsize += ancestor.adjustedVsize; + tx.ancestorSigops += ancestor.sigops; + }); + tx.score = tx.ancestorFee / tx.ancestorVsize; + tx.relativesSet = true; +} + +// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score +// avoids recursion to limit call stack depth +function updateDescendants( + rootTx: MinerTransaction, + mempool: Map, + modified: MinerTransaction[], + clusterRate: number, +): MinerTransaction[] { + const descendantSet: Set = new Set(); + // stack of nodes left to visit + const descendants: MinerTransaction[] = []; + let descendantTx: MinerTransaction | undefined; + rootTx.children.forEach(childTx => { + if (!descendantSet.has(childTx)) { + descendants.push(childTx); + descendantSet.add(childTx); + } + }); + while (descendants.length) { + descendantTx = descendants.pop(); + if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) { + // remove tx as ancestor + descendantTx.ancestorMap.delete(rootTx.txid); + descendantTx.ancestorFee -= (rootTx.fee + rootTx.feeDelta); + descendantTx.ancestorVsize -= rootTx.adjustedVsize; + descendantTx.ancestorSigops -= rootTx.sigops; + descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorVsize; + descendantTx.dependencyRate = descendantTx.dependencyRate ? Math.min(descendantTx.dependencyRate, clusterRate) : clusterRate; + + if (!descendantTx.modified) { + descendantTx.modified = true; + modified.push(descendantTx); + } + + // add this node's children to the stack + descendantTx.children.forEach(childTx => { + // visit each node only once + if (!descendantSet.has(childTx)) { + descendants.push(childTx); + descendantSet.add(childTx); + } + }); + } + } + // return new, resorted modified list + return modified.sort(priorityComparator); +} + +// Used to sort an array of MinerTransactions by descending ancestor score +function priorityComparator(a: MinerTransaction, b: MinerTransaction): number { + if (b.score === a.score) { + // tie-break by txid for stability + return a.order - b.order; + } else { + return b.score - a.score; + } +} + +// returns the most significant 4 bytes of the txid as an integer +function txidToOrdering(txid: string): number { + return parseInt( + txid.substring(62, 64) + + txid.substring(60, 62) + + txid.substring(58, 60) + + txid.substring(56, 58), + 16 + ); +} diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 9a5eb310a..539316e39 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 68; + private static currentVersion = 69; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -580,6 +580,11 @@ class DatabaseMigration { await this.$executeQuery(`INSERT INTO state VALUES('last_bitcoin_block_audit', 0, NULL);`); await this.updateToSchemaVersion(68); } + + if (databaseSchemaVersion < 69 && config.MEMPOOL.NETWORK === 'mainnet') { + await this.$executeQuery(this.getCreateAccelerationsTableQuery(), await this.$checkIfTableExists('accelerations')); + await this.updateToSchemaVersion(69); + } } /** @@ -1123,6 +1128,23 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateAccelerationsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS accelerations ( + txid varchar(65) NOT NULL, + added date NOT NULL, + height int(10) NOT NULL, + pool smallint unsigned NULL, + effective_vsize int(10) NOT NULL, + effective_fee bigint(20) unsigned NOT NULL, + boost_rate float unsigned, + boost_cost bigint(20) unsigned NOT NULL, + PRIMARY KEY (txid), + INDEX (added), + INDEX (height), + INDEX (pool) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + public async $blocksReindexingTruncate(): Promise { logger.warn(`Truncating pools, blocks, hashrates and difficulty_adjustments tables for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`); await Common.sleep$(5000); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index b78389b64..5c007fadd 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -24,6 +24,8 @@ import { ApiPrice } from '../repositories/PricesRepository'; import accelerationApi from './services/acceleration'; import mempool from './mempool'; import statistics from './statistics/statistics'; +import accelerationCosts from './acceleration'; +import accelerationRepository from '../repositories/AccelerationRepository'; interface AddressTransactions { mempool: MempoolTransactionExtended[], @@ -728,6 +730,28 @@ class WebsocketHandler { const _memPool = memPool.getMempool(); + const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); + + + if (isAccelerated) { + const blockTxs: { [txid: string]: MempoolTransactionExtended } = {}; + for (const tx of transactions) { + blockTxs[tx.txid] = tx; + } + const accelerations = Object.values(mempool.getAccelerations()); + const boostRate = accelerationCosts.calculateBoostRate( + accelerations.map(acc => ({ txid: acc.txid, max_bid: acc.feeDelta })), + transactions + ); + for (const acc of accelerations) { + if (blockTxs[acc.txid]) { + const tx = blockTxs[acc.txid]; + const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions); + accelerationRepository.$saveAcceleration(accelerationInfo, block, block.extras.pool.id); + } + } + } + const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap()); memPool.handleMinedRbfTransactions(rbfTransactions); memPool.removeFromSpendMap(transactions); @@ -735,7 +759,6 @@ class WebsocketHandler { if (config.MEMPOOL.AUDIT && memPool.isInSync()) { let projectedBlocks; let auditMempool = _memPool; - const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; diff --git a/backend/src/repositories/AccelerationRepository.ts b/backend/src/repositories/AccelerationRepository.ts new file mode 100644 index 000000000..9fbed4673 --- /dev/null +++ b/backend/src/repositories/AccelerationRepository.ts @@ -0,0 +1,43 @@ +import { AccelerationInfo } from '../api/acceleration'; +import { ResultSetHeader, RowDataPacket } from 'mysql2'; +import DB from '../database'; +import logger from '../logger'; +import { IEsploraApi } from '../api/bitcoin/esplora-api.interface'; + +class AccelerationRepository { + public async $saveAcceleration(acceleration: AccelerationInfo, block: IEsploraApi.Block, pool_id: number): Promise { + try { + await DB.query(` + INSERT INTO accelerations(txid, added, height, pool, effective_vsize, effective_fee, boost_rate, boost_cost) + VALUE (?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + added = FROM_UNIXTIME(?), + height = ?, + pool = ?, + effective_vsize = ?, + effective_fee = ?, + boost_rate = ?, + boost_cost = ? + `, [ + acceleration.txSummary.txid, + block.timestamp, + block.height, + pool_id, + acceleration.txSummary.effectiveVsize, + acceleration.txSummary.effectiveFee, + acceleration.targetFeeRate, acceleration.cost, + block.timestamp, + block.height, + pool_id, + acceleration.txSummary.effectiveVsize, + acceleration.txSummary.effectiveFee, + acceleration.targetFeeRate, acceleration.cost, + ]); + } catch (e: any) { + logger.err(`Cannot save acceleration (${acceleration.txSummary.txid}) into db. Reason: ` + (e instanceof Error ? e.message : e)); + // We don't throw, not a critical issue if we miss some accelerations + } + } +} + +export default new AccelerationRepository();