From 3eec9cb0bb1c8e78bfc9a3729539d2c087bd2022 Mon Sep 17 00:00:00 2001 From: Andrew Camilleri Date: Thu, 18 Jan 2024 07:27:19 +0100 Subject: [PATCH] Refactor fee provider (#5643) * Refactor fee provider The fee provider ended up glued with a hardcoded factory. This PR: * removes this glue and uses the DI to register fee provider for a network. (allows plugins to add their own fee providers, for any network * Add a 10 second timeout to mempoolspace fee fetching as they are slow at times * use linear interpolation for mempool space fee estimation * fix upper bound * Add tests, rollback pluginify FeeProvider --------- Co-authored-by: nicolas.dorier --- BTCPayServer.Tests/FastTests.cs | 33 +++++ BTCPayServer.Tests/ThirdPartyTests.cs | 34 ++++- .../Services/Fees/MempoolSpaceFeeProvider.cs | 126 ++++++++++++------ 3 files changed, 150 insertions(+), 43 deletions(-) diff --git a/BTCPayServer.Tests/FastTests.cs b/BTCPayServer.Tests/FastTests.cs index 9ec8fb862..cbe84c61d 100644 --- a/BTCPayServer.Tests/FastTests.cs +++ b/BTCPayServer.Tests/FastTests.cs @@ -28,6 +28,7 @@ using BTCPayServer.Plugins.Bitcoin; using BTCPayServer.Rating; using BTCPayServer.Services; using BTCPayServer.Services.Apps; +using BTCPayServer.Services.Fees; using BTCPayServer.Services.Invoices; using BTCPayServer.Services.Labels; using BTCPayServer.Services.Rates; @@ -159,6 +160,38 @@ namespace BTCPayServer.Tests Assert.Equal("Test", data.FromAsset); } + [Fact] + public void CanInterpolateOrBound() + { + var testData = new ((int Blocks, decimal Fee)[] Data, int Target, decimal Expected) [] + { + ([(0, 0m), (10, 100m)], 5, 50m), + ([(50, 0m), (100, 100m)], 5, 0.0m), + ([(50, 0m), (100, 100m)], 101, 100.0m), + ([(50, 100m), (50, 100m)], 101, 100.0m), + ([(50, 0m), (100, 100m)], 75, 50m), + ([(0, 0m), (50, 50m), (100, 100m)], 75, 75m), + ([(0, 0m), (500, 50m), (1000, 100m)], 750, 75m), + ([(0, 0m), (500, 50m), (1000, 100m)], 100, 10m), + ([(0, 0m), (100, 100m)], 80, 80m), + ([(0, 0m), (100, 100m)], 25, 25m), + }; + foreach (var t in testData) + { + var actual = MempoolSpaceFeeProvider.InterpolateOrBound(t.Data.Select(t => new MempoolSpaceFeeProvider.BlockFeeRate(t.Blocks, new FeeRate(t.Fee))).ToArray(), t.Target); + Assert.Equal(new FeeRate(t.Expected), actual); + } + } + [Fact] + public void CanRandomizeByPercentage() + { + var generated = Enumerable.Range(0, 1000).Select(_ => MempoolSpaceFeeProvider.RandomizeByPercentage(100.0m, 10.0m)).ToArray(); + Assert.Empty(generated.Where(g => g < 90m)); + Assert.Empty(generated.Where(g => g > 110m)); + Assert.NotEmpty(generated.Where(g => g < 91m)); + Assert.NotEmpty(generated.Where(g => g > 109m)); + } + private void CanParseDecimalsCore(string str, decimal expected) { var d = JsonConvert.DeserializeObject(str); diff --git a/BTCPayServer.Tests/ThirdPartyTests.cs b/BTCPayServer.Tests/ThirdPartyTests.cs index a98d55435..6d3e5cb3e 100644 --- a/BTCPayServer.Tests/ThirdPartyTests.cs +++ b/BTCPayServer.Tests/ThirdPartyTests.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using BTCPayServer.Controllers; using BTCPayServer.Data; using BTCPayServer.Models.StoreViewModels; +using BTCPayServer.Models.WalletViewModels; using BTCPayServer.Rating; using BTCPayServer.Services.Fees; using BTCPayServer.Services.Rates; @@ -92,7 +93,38 @@ namespace BTCPayServer.Tests isTestnet); var rates = await mempoolSpaceFeeProvider.GetFeeRatesAsync(); Assert.NotEmpty(rates); - await mempoolSpaceFeeProvider.GetFeeRateAsync(20); + + + var recommendedFees = + await Task.WhenAll(new[] + { + TimeSpan.FromMinutes(10.0), TimeSpan.FromMinutes(60.0), TimeSpan.FromHours(6.0), + TimeSpan.FromHours(24.0), + }.Select(async time => + { + try + { + var result = await mempoolSpaceFeeProvider.GetFeeRateAsync( + (int)Network.Main.Consensus.GetExpectedBlocksFor(time)); + return new WalletSendModel.FeeRateOption() + { + Target = time, + FeeRate = result.SatoshiPerByte + }; + } + catch (Exception) + { + return null; + } + }) + .ToArray()); + //ENSURE THESE ARE LOGICAL + Assert.True(recommendedFees[0].FeeRate >= recommendedFees[1].FeeRate, $"{recommendedFees[0].Target}:{recommendedFees[0].FeeRate} >= {recommendedFees[1].Target}:{recommendedFees[1].FeeRate}"); + Assert.True(recommendedFees[1].FeeRate >= recommendedFees[2].FeeRate, $"{recommendedFees[1].Target}:{recommendedFees[1].FeeRate} >= {recommendedFees[2].Target}:{recommendedFees[2].FeeRate}"); + Assert.True(recommendedFees[2].FeeRate >= recommendedFees[3].FeeRate, $"{recommendedFees[2].Target}:{recommendedFees[2].FeeRate} >= {recommendedFees[3].Target}:{recommendedFees[3].FeeRate}"); + + + } } [Fact] diff --git a/BTCPayServer/Services/Fees/MempoolSpaceFeeProvider.cs b/BTCPayServer/Services/Fees/MempoolSpaceFeeProvider.cs index 85cd5d582..90e53db9f 100644 --- a/BTCPayServer/Services/Fees/MempoolSpaceFeeProvider.cs +++ b/BTCPayServer/Services/Fees/MempoolSpaceFeeProvider.cs @@ -3,21 +3,26 @@ using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using AngleSharp.Dom; +using ExchangeSharp; using Microsoft.Extensions.Caching.Memory; using NBitcoin; +using Org.BouncyCastle.Asn1.X509; +using YamlDotNet.Core.Tokens; namespace BTCPayServer.Services.Fees; public class MempoolSpaceFeeProvider( - IMemoryCache MemoryCache, - string CacheKey, - IHttpClientFactory HttpClientFactory, - bool Testnet) : IFeeProvider + IMemoryCache memoryCache, + string cacheKey, + IHttpClientFactory httpClientFactory, + bool testnet) : IFeeProvider { - private readonly string ExplorerLink = Testnet switch + private string ExplorerLink = testnet switch { true => "https://mempool.space/testnet/api/v1/fees/recommended", false => "https://mempool.space/api/v1/fees/recommended" @@ -27,50 +32,87 @@ public class MempoolSpaceFeeProvider( { var result = await GetFeeRatesAsync(); - return result.TryGetValue(blockTarget, out var feeRate) - ? feeRate - : - //try get the closest one - result[result.Keys.MinBy(key => Math.Abs(key - blockTarget))]; - } - - public Task> GetFeeRatesAsync() - { - return MemoryCache.GetOrCreateAsync(CacheKey, async entry => - { - entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(5); - var client = HttpClientFactory.CreateClient(nameof(MempoolSpaceFeeProvider)); - using var result = await client.GetAsync(ExplorerLink); - result.EnsureSuccessStatusCode(); - var recommendedFees = await result.Content.ReadAsAsync>(); - var feesByBlockTarget = new Dictionary(); - foreach ((var feeId, decimal value) in recommendedFees) - { - var target = feeId switch - { - "fastestFee" => 1, - "halfHourFee" => 3, - "hourFee" => 6, - "economyFee" when recommendedFees.TryGetValue("minimumFee", out var minFee) && minFee == value => 144, - "economyFee" => 72, - "minimumFee" => 144, - _ => -1 - }; - feesByBlockTarget.TryAdd(target, new FeeRate(RandomizeByPercentage(value, 10))); - } - return feesByBlockTarget; - })!; + return InterpolateOrBound(result, blockTarget); + } - static decimal RandomizeByPercentage(decimal value, int percentage) + internal static FeeRate InterpolateOrBound(BlockFeeRate[] ordered, int target) { - decimal range = value * percentage / 100m; - var res = value + range * (Random.Shared.NextDouble() < 0.5 ? -1 : 1); + (BlockFeeRate lb, BlockFeeRate hb) = (ordered[0], ordered[^1]); + target = Math.Clamp(target, lb.Blocks, hb.Blocks); + for (int i = 0; i < ordered.Length; i++) + { + if (ordered[i].Blocks > lb.Blocks && ordered[i].Blocks <= target) + lb = ordered[i]; + if (ordered[i].Blocks < hb.Blocks && ordered[i].Blocks >= target) + hb = ordered[i]; + } + if (hb.Blocks == lb.Blocks) + return hb.FeeRate; + var a = (decimal)(target - lb.Blocks) / (decimal)(hb.Blocks - lb.Blocks); + return new FeeRate((1 - a) * lb.FeeRate.SatoshiPerByte + a * hb.FeeRate.SatoshiPerByte); + } + internal async Task GetFeeRatesAsync() + { + try + { + return (await memoryCache.GetOrCreateAsync(cacheKey, async entry => + { + entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(5); + return await GetFeeRatesCore(); + }))!; + } + catch (Exception) + { + memoryCache.Remove(cacheKey); + throw; + } + } + internal record BlockFeeRate(int Blocks, FeeRate FeeRate); + async Task GetFeeRatesCore() + { + var client = httpClientFactory.CreateClient(nameof(MempoolSpaceFeeProvider)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + using var result = await client.GetAsync(ExplorerLink, cts.Token); + result.EnsureSuccessStatusCode(); + var recommendedFees = await result.Content.ReadAsAsync>(); + var r = new List(); + foreach ((var feeId, decimal value) in recommendedFees) + { + var target = feeId switch + { + "fastestFee" => 1, + "halfHourFee" => 3, + "hourFee" => 6, + "economyFee" when recommendedFees.TryGetValue("minimumFee", out var minFee) && minFee == value => 144, + "economyFee" => 72, + "minimumFee" => 144, + _ => -1 + }; + r.Add(new(target, new FeeRate(value))); + } + var ordered = r.OrderBy(k => k.Blocks).ToArray(); + for (var i = 0; i < ordered.Length; i++) + { + // Randomize a bit + ordered[i] = ordered[i] with { FeeRate = new FeeRate(RandomizeByPercentage(ordered[i].FeeRate.SatoshiPerByte, 10m)) }; + if (i > 0) // Make sure feerate always increase + ordered[i] = ordered[i] with { FeeRate = FeeRate.Max(ordered[i - 1].FeeRate, ordered[i].FeeRate) }; + } + return ordered; + } + + internal static decimal RandomizeByPercentage(decimal value, decimal percentage) + { + if (value is 1) + return 1; + decimal range = (value * percentage) / 100m; + var res = value + (range * 2.0m) * ((decimal)(Random.Shared.NextDouble() - 0.5)); return res switch { < 1m => 1m, - > 850m => 2000m, + > 2000m => 2000m, _ => res }; }