From 2baa10dcefe3cb49d6cd7cb2f89835b0a19eade1 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 12 Mar 2023 11:09:11 +0900 Subject: [PATCH] Use effective fee rate heuristics for block fee span --- backend/src/api/blocks.ts | 103 ++++++++++++++++-- backend/src/mempool.interfaces.ts | 10 ++ backend/src/repositories/BlocksRepository.ts | 21 +++- backend/src/repositories/CpfpRepository.ts | 2 +- .../blockchain-blocks.component.html | 2 +- 5 files changed, 124 insertions(+), 14 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 755b3c3ad..72dd7fa0f 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,7 +2,7 @@ import config from '../config'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo, EffectiveFeeStats, CpfpSummary } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; @@ -200,8 +200,15 @@ class Blocks { extras.segwitTotalWeight = 0; } else { const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id); - extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles - extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); + let feeStats = { + medianFee: stats.feerate_percentiles[2], // 50th percentiles + feeRange: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(), + }; + if (transactions?.length > 1) { + feeStats = this.calcEffectiveFeeStatistics(transactions); + } + extras.medianFee = feeStats.medianFee; + extras.feeRange = feeStats.feeRange; extras.totalFees = stats.totalfee; extras.avgFee = stats.avgfee; extras.avgFeeRate = stats.avgfeerate; @@ -571,7 +578,8 @@ class Blocks { const block = BitcoinApi.convertBlock(verboseBlock); const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, false); - const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions); + const cpfpSummary: CpfpSummary = this.calculateCpfp(block.height, transactions); + const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions); const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock); // start async callbacks @@ -619,7 +627,7 @@ class Blocks { await this.$getStrippedBlockTransactions(blockExtended.id, true); } if (config.MEMPOOL.CPFP_INDEXING) { - this.$indexCPFP(blockExtended.id, this.currentBlockHeight); + this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary); } } } @@ -913,33 +921,45 @@ class Blocks { public async $indexCPFP(hash: string, height: number): Promise { const block = await bitcoinClient.getBlock(hash, 2); const transactions = block.tx.map(tx => { - tx.vsize = tx.weight / 4; tx.fee *= 100_000_000; return tx; }); - const clusters: any[] = []; + const summary = this.calculateCpfp(height, transactions); - let cluster: TransactionStripped[] = []; + await this.$saveCpfp(hash, height, summary); + + const effectiveFeeStats = this.calcEffectiveFeeStatistics(summary.transactions); + await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); + } + + public calculateCpfp(height: number, transactions: TransactionExtended[]): CpfpSummary { + const clusters: any[] = []; + let cluster: TransactionExtended[] = []; let ancestors: { [txid: string]: boolean } = {}; + const txMap = {}; for (let i = transactions.length - 1; i >= 0; i--) { const tx = transactions[i]; + txMap[tx.txid] = tx; if (!ancestors[tx.txid]) { let totalFee = 0; let totalVSize = 0; cluster.forEach(tx => { totalFee += tx?.fee || 0; - totalVSize += tx.vsize; + totalVSize += (tx.weight / 4); }); const effectiveFeePerVsize = totalFee / totalVSize; if (cluster.length > 1) { clusters.push({ root: cluster[0].txid, height, - txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), + txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }; }), effectiveFeePerVsize, }); } + cluster.forEach(tx => { + txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize; + }) cluster = []; ancestors = {}; } @@ -948,11 +968,72 @@ class Blocks { ancestors[vin.txid] = true; }); } - const result = await cpfpRepository.$batchSaveClusters(clusters); + return { + transactions, + clusters, + }; + } + + public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise { + const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters); if (!result) { await cpfpRepository.$insertProgressMarker(height); } } + + private calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats { + const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate); + const halfTotalWeight = transactions.reduce((total, tx) => total += tx.weight, 0) / 2; + let weightCount = 0; + let medianFee = 0; + let medianWeight = 0; + + // calculate the "medianFee" as the weighted-average fee rate of the middle 10000 weight units of transactions + const leftBound = halfTotalWeight - 5000; + const rightBound = halfTotalWeight + 5000; + for (let i = 0; i < sortedTxs.length && weightCount < rightBound; i++) { + const left = weightCount; + const right = weightCount + sortedTxs[i].weight; + if (right > leftBound) { + const weight = Math.min(right, rightBound) - Math.max(left, leftBound); + medianFee += (sortedTxs[i].rate * (weight / 4) ); + medianWeight += weight; + } + weightCount += sortedTxs[i].weight; + } + const medianFeeRate = medianFee / (medianWeight / 4); + + // minimum effective fee heuristic: + // lowest of + // a) the 1st percentile of effective fee rates + // b) the minimum effective fee rate in the last 2% of transactions (in block order) + const minFee = Math.min( + getNthPercentile(1, sortedTxs).rate, + transactions.slice(-transactions.length / 50).reduce((min, tx) => { return Math.min(min, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, Infinity) + ); + + // maximum effective fee heuristic: + // highest of + // a) the 99th percentile of effective fee rates + // b) the maximum effective fee rate in the first 2% of transactions (in block order) + const maxFee = Math.max( + getNthPercentile(99, sortedTxs).rate, + transactions.slice(0, transactions.length / 50).reduce((max, tx) => { return Math.max(max, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, 0) + ); + + return { + medianFee: medianFeeRate, + feeRange: [ + minFee, + [10,25,50,75,90].map(n => getNthPercentile(n, sortedTxs).rate), + maxFee, + ].flat(), + }; + } +} + +function getNthPercentile(n: number, sortedDistribution: any[]): any { + return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))]; } export default new Blocks(); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 28c1e21b5..16b856bcc 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -214,6 +214,16 @@ export interface MempoolStats { tx_count: number; } +export interface EffectiveFeeStats { + medianFee: number; // median effective fee rate + feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles +} + +export interface CpfpSummary { + transactions: TransactionExtended[]; + clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[]; +} + export interface Statistic { id?: number; added: string; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 04dcd4b56..4758c0708 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,4 +1,4 @@ -import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces'; import DB from '../database'; import logger from '../logger'; import { Common } from '../api/common'; @@ -908,6 +908,25 @@ class BlocksRepository { } } + /** + * Save indexed effective fee statistics + * + * @param id + * @param feeStats + */ + public async $saveEffectiveFeeStats(id: string, feeStats: EffectiveFeeStats): Promise { + try { + await DB.query(` + UPDATE blocks SET median_fee = ?, fee_span = ? + WHERE hash = ?`, + [feeStats.medianFee, JSON.stringify(feeStats.feeRange), id] + ); + } catch (e) { + logger.err(`Cannot update block fee stats. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + /** * Convert a mysql row block into a BlockExtended. Note that you * must provide the correct field into dbBlk object param diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts index 9d8b2fe75..b68c25472 100644 --- a/backend/src/repositories/CpfpRepository.ts +++ b/backend/src/repositories/CpfpRepository.ts @@ -48,7 +48,7 @@ class CpfpRepository { } } - public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise { + public async $batchSaveClusters(clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[]): Promise { try { const clusterValues: any[] = []; const txs: any[] = []; diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html index 6cdb895ff..373605667 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html @@ -25,7 +25,7 @@
- {{ block?.extras?.feeRange?.[1] | number:feeRounding }} - {{ + {{ block?.extras?.feeRange?.[0] | number:feeRounding }} - {{ block?.extras?.feeRange[block?.extras?.feeRange?.length - 1] | number:feeRounding }} sat/vB