From 2baa10dcefe3cb49d6cd7cb2f89835b0a19eade1 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 12 Mar 2023 11:09:11 +0900 Subject: [PATCH 1/2] Use effective fee rate heuristics for block fee span --- backend/src/api/blocks.ts | 103 ++++++++++++++++-- backend/src/mempool.interfaces.ts | 10 ++ backend/src/repositories/BlocksRepository.ts | 21 +++- backend/src/repositories/CpfpRepository.ts | 2 +- .../blockchain-blocks.component.html | 2 +- 5 files changed, 124 insertions(+), 14 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 755b3c3ad..72dd7fa0f 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,7 +2,7 @@ import config from '../config'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo, EffectiveFeeStats, CpfpSummary } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; @@ -200,8 +200,15 @@ class Blocks { extras.segwitTotalWeight = 0; } else { const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id); - extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles - extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); + let feeStats = { + medianFee: stats.feerate_percentiles[2], // 50th percentiles + feeRange: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(), + }; + if (transactions?.length > 1) { + feeStats = this.calcEffectiveFeeStatistics(transactions); + } + extras.medianFee = feeStats.medianFee; + extras.feeRange = feeStats.feeRange; extras.totalFees = stats.totalfee; extras.avgFee = stats.avgfee; extras.avgFeeRate = stats.avgfeerate; @@ -571,7 +578,8 @@ class Blocks { const block = BitcoinApi.convertBlock(verboseBlock); const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, false); - const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions); + const cpfpSummary: CpfpSummary = this.calculateCpfp(block.height, transactions); + const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions); const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock); // start async callbacks @@ -619,7 +627,7 @@ class Blocks { await this.$getStrippedBlockTransactions(blockExtended.id, true); } if (config.MEMPOOL.CPFP_INDEXING) { - this.$indexCPFP(blockExtended.id, this.currentBlockHeight); + this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary); } } } @@ -913,33 +921,45 @@ class Blocks { public async $indexCPFP(hash: string, height: number): Promise { const block = await bitcoinClient.getBlock(hash, 2); const transactions = block.tx.map(tx => { - tx.vsize = tx.weight / 4; tx.fee *= 100_000_000; return tx; }); - const clusters: any[] = []; + const summary = this.calculateCpfp(height, transactions); - let cluster: TransactionStripped[] = []; + await this.$saveCpfp(hash, height, summary); + + const effectiveFeeStats = this.calcEffectiveFeeStatistics(summary.transactions); + await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); + } + + public calculateCpfp(height: number, transactions: TransactionExtended[]): CpfpSummary { + const clusters: any[] = []; + let cluster: TransactionExtended[] = []; let ancestors: { [txid: string]: boolean } = {}; + const txMap = {}; for (let i = transactions.length - 1; i >= 0; i--) { const tx = transactions[i]; + txMap[tx.txid] = tx; if (!ancestors[tx.txid]) { let totalFee = 0; let totalVSize = 0; cluster.forEach(tx => { totalFee += tx?.fee || 0; - totalVSize += tx.vsize; + totalVSize += (tx.weight / 4); }); const effectiveFeePerVsize = totalFee / totalVSize; if (cluster.length > 1) { clusters.push({ root: cluster[0].txid, height, - txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), + txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }; }), effectiveFeePerVsize, }); } + cluster.forEach(tx => { + txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize; + }) cluster = []; ancestors = {}; } @@ -948,11 +968,72 @@ class Blocks { ancestors[vin.txid] = true; }); } - const result = await cpfpRepository.$batchSaveClusters(clusters); + return { + transactions, + clusters, + }; + } + + public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise { + const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters); if (!result) { await cpfpRepository.$insertProgressMarker(height); } } + + private calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats { + const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate); + const halfTotalWeight = transactions.reduce((total, tx) => total += tx.weight, 0) / 2; + let weightCount = 0; + let medianFee = 0; + let medianWeight = 0; + + // calculate the "medianFee" as the weighted-average fee rate of the middle 10000 weight units of transactions + const leftBound = halfTotalWeight - 5000; + const rightBound = halfTotalWeight + 5000; + for (let i = 0; i < sortedTxs.length && weightCount < rightBound; i++) { + const left = weightCount; + const right = weightCount + sortedTxs[i].weight; + if (right > leftBound) { + const weight = Math.min(right, rightBound) - Math.max(left, leftBound); + medianFee += (sortedTxs[i].rate * (weight / 4) ); + medianWeight += weight; + } + weightCount += sortedTxs[i].weight; + } + const medianFeeRate = medianFee / (medianWeight / 4); + + // minimum effective fee heuristic: + // lowest of + // a) the 1st percentile of effective fee rates + // b) the minimum effective fee rate in the last 2% of transactions (in block order) + const minFee = Math.min( + getNthPercentile(1, sortedTxs).rate, + transactions.slice(-transactions.length / 50).reduce((min, tx) => { return Math.min(min, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, Infinity) + ); + + // maximum effective fee heuristic: + // highest of + // a) the 99th percentile of effective fee rates + // b) the maximum effective fee rate in the first 2% of transactions (in block order) + const maxFee = Math.max( + getNthPercentile(99, sortedTxs).rate, + transactions.slice(0, transactions.length / 50).reduce((max, tx) => { return Math.max(max, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, 0) + ); + + return { + medianFee: medianFeeRate, + feeRange: [ + minFee, + [10,25,50,75,90].map(n => getNthPercentile(n, sortedTxs).rate), + maxFee, + ].flat(), + }; + } +} + +function getNthPercentile(n: number, sortedDistribution: any[]): any { + return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))]; } export default new Blocks(); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 28c1e21b5..16b856bcc 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -214,6 +214,16 @@ export interface MempoolStats { tx_count: number; } +export interface EffectiveFeeStats { + medianFee: number; // median effective fee rate + feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles +} + +export interface CpfpSummary { + transactions: TransactionExtended[]; + clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[]; +} + export interface Statistic { id?: number; added: string; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 04dcd4b56..4758c0708 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,4 +1,4 @@ -import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces'; import DB from '../database'; import logger from '../logger'; import { Common } from '../api/common'; @@ -908,6 +908,25 @@ class BlocksRepository { } } + /** + * Save indexed effective fee statistics + * + * @param id + * @param feeStats + */ + public async $saveEffectiveFeeStats(id: string, feeStats: EffectiveFeeStats): Promise { + try { + await DB.query(` + UPDATE blocks SET median_fee = ?, fee_span = ? + WHERE hash = ?`, + [feeStats.medianFee, JSON.stringify(feeStats.feeRange), id] + ); + } catch (e) { + logger.err(`Cannot update block fee stats. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + /** * Convert a mysql row block into a BlockExtended. Note that you * must provide the correct field into dbBlk object param diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts index 9d8b2fe75..b68c25472 100644 --- a/backend/src/repositories/CpfpRepository.ts +++ b/backend/src/repositories/CpfpRepository.ts @@ -48,7 +48,7 @@ class CpfpRepository { } } - public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise { + public async $batchSaveClusters(clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[]): Promise { try { const clusterValues: any[] = []; const txs: any[] = []; diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html index 6cdb895ff..373605667 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html @@ -25,7 +25,7 @@
- {{ block?.extras?.feeRange?.[1] | number:feeRounding }} - {{ + {{ block?.extras?.feeRange?.[0] | number:feeRounding }} - {{ block?.extras?.feeRange[block?.extras?.feeRange?.length - 1] | number:feeRounding }} sat/vB
From 2fc404a55c4486805457c75ef202451bf801625d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 12 Mar 2023 14:36:36 +0900 Subject: [PATCH 2/2] refactor effective rate calculation --- backend/src/api/blocks.ts | 105 ++------------------------------------ backend/src/api/common.ts | 97 ++++++++++++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 101 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 72dd7fa0f..eee5dae6e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,7 +2,7 @@ import config from '../config'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo, EffectiveFeeStats, CpfpSummary } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo, CpfpSummary } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; @@ -205,7 +205,7 @@ class Blocks { feeRange: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(), }; if (transactions?.length > 1) { - feeStats = this.calcEffectiveFeeStatistics(transactions); + feeStats = Common.calcEffectiveFeeStatistics(transactions); } extras.medianFee = feeStats.medianFee; extras.feeRange = feeStats.feeRange; @@ -578,7 +578,7 @@ class Blocks { const block = BitcoinApi.convertBlock(verboseBlock); const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, false); - const cpfpSummary: CpfpSummary = this.calculateCpfp(block.height, transactions); + const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions); const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions); const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock); @@ -925,115 +925,20 @@ class Blocks { return tx; }); - const summary = this.calculateCpfp(height, transactions); + const summary = Common.calculateCpfp(height, transactions); await this.$saveCpfp(hash, height, summary); - const effectiveFeeStats = this.calcEffectiveFeeStatistics(summary.transactions); + const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); } - public calculateCpfp(height: number, transactions: TransactionExtended[]): CpfpSummary { - const clusters: any[] = []; - let cluster: TransactionExtended[] = []; - let ancestors: { [txid: string]: boolean } = {}; - const txMap = {}; - for (let i = transactions.length - 1; i >= 0; i--) { - const tx = transactions[i]; - txMap[tx.txid] = tx; - if (!ancestors[tx.txid]) { - let totalFee = 0; - let totalVSize = 0; - cluster.forEach(tx => { - totalFee += tx?.fee || 0; - totalVSize += (tx.weight / 4); - }); - const effectiveFeePerVsize = totalFee / totalVSize; - if (cluster.length > 1) { - clusters.push({ - root: cluster[0].txid, - height, - txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }; }), - effectiveFeePerVsize, - }); - } - cluster.forEach(tx => { - txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize; - }) - cluster = []; - ancestors = {}; - } - cluster.push(tx); - tx.vin.forEach(vin => { - ancestors[vin.txid] = true; - }); - } - return { - transactions, - clusters, - }; - } - public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise { const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters); if (!result) { await cpfpRepository.$insertProgressMarker(height); } } - - private calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats { - const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate); - const halfTotalWeight = transactions.reduce((total, tx) => total += tx.weight, 0) / 2; - let weightCount = 0; - let medianFee = 0; - let medianWeight = 0; - - // calculate the "medianFee" as the weighted-average fee rate of the middle 10000 weight units of transactions - const leftBound = halfTotalWeight - 5000; - const rightBound = halfTotalWeight + 5000; - for (let i = 0; i < sortedTxs.length && weightCount < rightBound; i++) { - const left = weightCount; - const right = weightCount + sortedTxs[i].weight; - if (right > leftBound) { - const weight = Math.min(right, rightBound) - Math.max(left, leftBound); - medianFee += (sortedTxs[i].rate * (weight / 4) ); - medianWeight += weight; - } - weightCount += sortedTxs[i].weight; - } - const medianFeeRate = medianFee / (medianWeight / 4); - - // minimum effective fee heuristic: - // lowest of - // a) the 1st percentile of effective fee rates - // b) the minimum effective fee rate in the last 2% of transactions (in block order) - const minFee = Math.min( - getNthPercentile(1, sortedTxs).rate, - transactions.slice(-transactions.length / 50).reduce((min, tx) => { return Math.min(min, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, Infinity) - ); - - // maximum effective fee heuristic: - // highest of - // a) the 99th percentile of effective fee rates - // b) the maximum effective fee rate in the first 2% of transactions (in block order) - const maxFee = Math.max( - getNthPercentile(99, sortedTxs).rate, - transactions.slice(0, transactions.length / 50).reduce((max, tx) => { return Math.max(max, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, 0) - ); - - return { - medianFee: medianFeeRate, - feeRange: [ - minFee, - [10,25,50,75,90].map(n => getNthPercentile(n, sortedTxs).rate), - maxFee, - ].flat(), - }; - } -} - -function getNthPercentile(n: number, sortedDistribution: any[]): any { - return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))]; } export default new Blocks(); diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index df97c0292..1d3b11d66 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,4 +1,4 @@ -import { CpfpInfo, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces'; +import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces'; import config from '../config'; import { NodeSocket } from '../repositories/NodesSocketsRepository'; import { isIP } from 'net'; @@ -345,4 +345,99 @@ export class Common { }; } } + + static calculateCpfp(height: number, transactions: TransactionExtended[]): CpfpSummary { + const clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[] = []; + let cluster: TransactionExtended[] = []; + let ancestors: { [txid: string]: boolean } = {}; + const txMap = {}; + for (let i = transactions.length - 1; i >= 0; i--) { + const tx = transactions[i]; + txMap[tx.txid] = tx; + if (!ancestors[tx.txid]) { + let totalFee = 0; + let totalVSize = 0; + cluster.forEach(tx => { + totalFee += tx?.fee || 0; + totalVSize += (tx.weight / 4); + }); + const effectiveFeePerVsize = totalFee / totalVSize; + if (cluster.length > 1) { + clusters.push({ + root: cluster[0].txid, + height, + txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }; }), + effectiveFeePerVsize, + }); + } + cluster.forEach(tx => { + txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize; + }); + cluster = []; + ancestors = {}; + } + cluster.push(tx); + tx.vin.forEach(vin => { + ancestors[vin.txid] = true; + }); + } + return { + transactions, + clusters, + }; + } + + static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats { + const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate); + + let weightCount = 0; + let medianFee = 0; + let medianWeight = 0; + + // calculate the "medianFee" as the average fee rate of the middle 10000 weight units of transactions + const leftBound = 1995000; + const rightBound = 2005000; + for (let i = 0; i < sortedTxs.length && weightCount < rightBound; i++) { + const left = weightCount; + const right = weightCount + sortedTxs[i].weight; + if (right > leftBound) { + const weight = Math.min(right, rightBound) - Math.max(left, leftBound); + medianFee += (sortedTxs[i].rate * (weight / 4) ); + medianWeight += weight; + } + weightCount += sortedTxs[i].weight; + } + const medianFeeRate = medianWeight ? (medianFee / (medianWeight / 4)) : 0; + + // minimum effective fee heuristic: + // lowest of + // a) the 1st percentile of effective fee rates + // b) the minimum effective fee rate in the last 2% of transactions (in block order) + const minFee = Math.min( + Common.getNthPercentile(1, sortedTxs).rate, + transactions.slice(-transactions.length / 50).reduce((min, tx) => { return Math.min(min, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, Infinity) + ); + + // maximum effective fee heuristic: + // highest of + // a) the 99th percentile of effective fee rates + // b) the maximum effective fee rate in the first 2% of transactions (in block order) + const maxFee = Math.max( + Common.getNthPercentile(99, sortedTxs).rate, + transactions.slice(0, transactions.length / 50).reduce((max, tx) => { return Math.max(max, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, 0) + ); + + return { + medianFee: medianFeeRate, + feeRange: [ + minFee, + [10,25,50,75,90].map(n => Common.getNthPercentile(n, sortedTxs).rate), + maxFee, + ].flat(), + }; + } + + static getNthPercentile(n: number, sortedDistribution: any[]): any { + return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))]; + } }