Merge pull request #2170 from BlueWallet/mempool-fees-estimation

FIX: better fee estimation based on mempool (closes #2154)

blue_modules/BlueElectrum.js

@@ -26,7 +26,8 @@ const hardcodedPeers = [
   { host: 'electrum3.bluewallet.io', ssl: '443' }, // 2x weight
 ];
 
-let mainClient: ElectrumClient = false;
+/** @type {ElectrumClient} */
+let mainClient;
 let mainConnected = false;
 let wasConnectedAtLeastOnce = false;
 let serverName = false;
@@ -459,10 +460,78 @@ module.exports.waitTillConnected = async function () {
   });
 };
 
+// Returns the value at a given percentile in a sorted numeric array.
+// "Linear interpolation between closest ranks" method
+function percentile(arr, p) {
+  if (arr.length === 0) return 0;
+  if (typeof p !== 'number') throw new TypeError('p must be a number');
+  if (p <= 0) return arr[0];
+  if (p >= 1) return arr[arr.length - 1];
+
+  const index = (arr.length - 1) * p;
+  const lower = Math.floor(index);
+  const upper = lower + 1;
+  const weight = index % 1;
+
+  if (upper >= arr.length) return arr[lower];
+  return arr[lower] * (1 - weight) + arr[upper] * weight;
+}
+
+/**
+ * The histogram is an array of [fee, vsize] pairs, where vsizen is the cumulative virtual size of mempool transactions
+ * with a fee rate in the interval [feen-1, feen], and feen-1 > feen.
+ *
+ * @param numberOfBlocks {Number}
+ * @param feeHistorgram {Array}
+ * @returns {number}
+ */
+module.exports.calcEstimateFeeFromFeeHistorgam = function (numberOfBlocks, feeHistorgram) {
+  // first, transforming histogram:
+  let totalVsize = 0;
+  const histogramToUse = [];
+  for (const h of feeHistorgram) {
+    let [fee, vsize] = h;
+    let timeToStop = false;
+
+    if (totalVsize + vsize >= 1000000 * numberOfBlocks) {
+      vsize = 1000000 * numberOfBlocks - totalVsize; // only the difference between current summarized sige to tip of the block
+      timeToStop = true;
+    }
+
+    histogramToUse.push({ fee, vsize });
+    totalVsize += vsize;
+    if (timeToStop) break;
+  }
+
+  // now we have histogram of precisely size for numberOfBlocks.
+  // lets spread it into flat array so its easier to calculate percentile:
+  let histogramFlat = [];
+  for (const hh of histogramToUse) {
+    histogramFlat = histogramFlat.concat(Array(Math.round(hh.vsize / 25000)).fill(hh.fee));
+    // division is needed so resulting flat array is not too huge
+  }
+
+  histogramFlat = histogramFlat.sort(function (a, b) {
+    return a - b;
+  });
+
+  return Math.round(percentile(histogramFlat, 0.5) || 1);
+};
+
 module.exports.estimateFees = async function () {
-  const fast = await module.exports.estimateFee(1);
-  const medium = await module.exports.estimateFee(18);
-  const slow = await module.exports.estimateFee(144);
+  const histogram = await mainClient.mempool_getFeeHistogram();
+
+  // fetching what electrum (which uses bitcoin core) thinks about fees:
+  const _fast = await module.exports.estimateFee(1);
+  const _medium = await module.exports.estimateFee(18);
+  const _slow = await module.exports.estimateFee(144);
+
+  // calculating fast fees from mempool:
+  const fast = module.exports.calcEstimateFeeFromFeeHistorgam(1, histogram);
+  // recalculating medium and slow fees using bitcoincore estimations only like relative weights:
+  // (minimum 1 sat, just for any case)
+  const medium = Math.max(1, Math.round((fast * _medium) / _fast));
+  const slow = Math.max(1, Math.round((fast * _slow) / _fast));
   return { fast, medium, slow };
 };
 

tests/integration/BlueElectrum.test.js

@@ -22,6 +22,75 @@ beforeAll(async () => {
 });
 
 describe('BlueElectrum', () => {
+  it('ElectrumClient can estimate fees from histogram', async () => {
+    assert.strictEqual(
+      BlueElectrum.calcEstimateFeeFromFeeHistorgam(1, [
+        [96, 105086],
+        [83, 124591],
+        [64, 108207],
+        [50, 131141],
+        [22, 148800],
+        [17, 156916],
+        [11, 413222],
+        [10, 361384],
+        [9, 294146],
+        [8, 121778],
+        [7, 1153727],
+        [6, 283925],
+        [5, 880946],
+        [4, 825703],
+        [3, 2179023],
+        [2, 590559],
+        [1, 1648473],
+      ]),
+      22,
+    );
+    assert.strictEqual(
+      BlueElectrum.calcEstimateFeeFromFeeHistorgam(18, [
+        [96, 105086],
+        [83, 124591],
+        [64, 108207],
+        [50, 131141],
+        [22, 148800],
+        [17, 156916],
+        [11, 413222],
+        [10, 361384],
+        [9, 294146],
+        [8, 121778],
+        [7, 1153727],
+        [6, 283925],
+        [5, 880946],
+        [4, 825703],
+        [3, 2179023],
+        [2, 590559],
+        [1, 1648473],
+      ]),
+      4,
+    );
+    assert.strictEqual(
+      BlueElectrum.calcEstimateFeeFromFeeHistorgam(144, [
+        [96, 105086],
+        [83, 124591],
+        [64, 108207],
+        [50, 131141],
+        [22, 148800],
+        [17, 156916],
+        [11, 413222],
+        [10, 361384],
+        [9, 294146],
+        [8, 121778],
+        [7, 1153727],
+        [6, 283925],
+        [5, 880946],
+        [4, 825703],
+        [3, 2179023],
+        [2, 590559],
+        [1, 1648473],
+      ]),
+      4,
+    );
+  });
+
   it('ElectrumClient can test connection', async () => {
     assert.ok(!(await BlueElectrum.testConnection('electrum1.bluewallet.io', 444, false)));
     assert.ok(!(await BlueElectrum.testConnection('electrum1.bluewallet.io', false, 444)));
@@ -41,6 +110,10 @@ describe('BlueElectrum', () => {
 
   it('ElectrumClient can estimate fees', async () => {
     assert.ok((await BlueElectrum.estimateFee(1)) > 1);
+    const fees = await BlueElectrum.estimateFees();
+    assert.ok(fees.fast > 0);
+    assert.ok(fees.medium > 0);
+    assert.ok(fees.slow > 0);
   });
 
   it('ElectrumClient can request server features', async () => {