Decouple RateProviderFactory with RateFetcher

This commit is contained in:
nicolas.dorier 2018-08-22 16:53:40 +09:00
parent 4f5a8f7953
commit 87d384dba5
12 changed files with 251 additions and 154 deletions

View file

@ -121,7 +121,7 @@ namespace BTCPayServer.Tests
_Host.Start();
InvoiceRepository = (InvoiceRepository)_Host.Services.GetService(typeof(InvoiceRepository));
StoreRepository = (StoreRepository)_Host.Services.GetService(typeof(StoreRepository));
var rateProvider = (BTCPayRateProviderFactory)_Host.Services.GetService(typeof(BTCPayRateProviderFactory));
var rateProvider = (RateProviderFactory)_Host.Services.GetService(typeof(RateProviderFactory));
rateProvider.DirectProviders.Clear();
var coinAverageMock = new MockRateProvider();

View file

@ -1666,8 +1666,7 @@ namespace BTCPayServer.Tests
[Fact]
public void CanQueryDirectProviders()
{
var provider = new BTCPayNetworkProvider(NetworkType.Mainnet);
var factory = CreateBTCPayRateFactory(provider);
var factory = CreateBTCPayRateFactory();
foreach (var result in factory
.DirectProviders
@ -1695,15 +1694,15 @@ namespace BTCPayServer.Tests
public void CanGetRateCryptoCurrenciesByDefault()
{
var provider = new BTCPayNetworkProvider(NetworkType.Mainnet);
var factory = CreateBTCPayRateFactory(provider);
var factory = CreateBTCPayRateFactory();
var fetcher = new RateFetcher(factory);
var pairs =
provider.GetAll()
.Select(c => new CurrencyPair(c.CryptoCode, "USD"))
.ToHashSet();
var rules = new StoreBlob().GetDefaultRateRules(provider);
var result = factory.FetchRates(pairs, rules);
var result = fetcher.FetchRates(pairs, rules);
foreach (var value in result)
{
var rateResult = value.Value.GetAwaiter().GetResult();
@ -1711,15 +1710,14 @@ namespace BTCPayServer.Tests
}
}
private static BTCPayRateProviderFactory CreateBTCPayRateFactory(BTCPayNetworkProvider provider)
private static RateProviderFactory CreateBTCPayRateFactory()
{
return new BTCPayRateProviderFactory(new MemoryCacheOptions() { ExpirationScanFrequency = TimeSpan.FromSeconds(1.0) }, null, provider, new CoinAverageSettings());
return new RateProviderFactory(new MemoryCacheOptions() { ExpirationScanFrequency = TimeSpan.FromSeconds(1.0) }, null, new CoinAverageSettings());
}
[Fact]
public void CheckRatesProvider()
{
var provider = new BTCPayNetworkProvider(NetworkType.Mainnet);
var coinAverage = new CoinAverageRateProvider();
var rates = coinAverage.GetRatesAsync().GetAwaiter().GetResult();
Assert.NotNull(rates.GetRate("coinaverage", new CurrencyPair("BTC", "JPY")));
@ -1728,27 +1726,28 @@ namespace BTCPayServer.Tests
RateRules.TryParse("X_X = coinaverage(X_X);", out var rateRules);
var factory = CreateBTCPayRateFactory(provider);
var factory = CreateBTCPayRateFactory();
var fetcher = new RateFetcher(CreateBTCPayRateFactory());
factory.CacheSpan = TimeSpan.FromSeconds(10);
var fetchedRate = factory.FetchRate(CurrencyPair.Parse("BTC_USD"), rateRules).GetAwaiter().GetResult();
var fetchedRate = fetcher.FetchRate(CurrencyPair.Parse("BTC_USD"), rateRules).GetAwaiter().GetResult();
Assert.False(fetchedRate.Cached);
fetchedRate = factory.FetchRate(CurrencyPair.Parse("BTC_USD"), rateRules).GetAwaiter().GetResult();
fetchedRate = fetcher.FetchRate(CurrencyPair.Parse("BTC_USD"), rateRules).GetAwaiter().GetResult();
Assert.True(fetchedRate.Cached);
Thread.Sleep(11000);
fetchedRate = factory.FetchRate(CurrencyPair.Parse("BTC_USD"), rateRules).GetAwaiter().GetResult();
fetchedRate = fetcher.FetchRate(CurrencyPair.Parse("BTC_USD"), rateRules).GetAwaiter().GetResult();
Assert.False(fetchedRate.Cached);
fetchedRate = factory.FetchRate(CurrencyPair.Parse("BTC_USD"), rateRules).GetAwaiter().GetResult();
fetchedRate = fetcher.FetchRate(CurrencyPair.Parse("BTC_USD"), rateRules).GetAwaiter().GetResult();
Assert.True(fetchedRate.Cached);
// Should cache at exchange level so this should hit the cache
var fetchedRate2 = factory.FetchRate(CurrencyPair.Parse("LTC_USD"), rateRules).GetAwaiter().GetResult();
var fetchedRate2 = fetcher.FetchRate(CurrencyPair.Parse("LTC_USD"), rateRules).GetAwaiter().GetResult();
Assert.True(fetchedRate.Cached);
Assert.NotEqual(fetchedRate.BidAsk.Bid, fetchedRate2.BidAsk.Bid);
// Should cache at exchange level this should not hit the cache as it is different exchange
RateRules.TryParse("X_X = bittrex(X_X);", out rateRules);
fetchedRate = factory.FetchRate(CurrencyPair.Parse("BTC_USD"), rateRules).GetAwaiter().GetResult();
fetchedRate = fetcher.FetchRate(CurrencyPair.Parse("BTC_USD"), rateRules).GetAwaiter().GetResult();
Assert.False(fetchedRate.Cached);
}

View file

@ -49,7 +49,7 @@ namespace BTCPayServer.Controllers
{
InvoiceRepository _InvoiceRepository;
ContentSecurityPolicies _CSP;
BTCPayRateProviderFactory _RateProvider;
RateFetcher _RateProvider;
StoreRepository _StoreRepository;
UserManager<ApplicationUser> _UserManager;
private CurrencyNameTable _CurrencyNameTable;
@ -62,7 +62,7 @@ namespace BTCPayServer.Controllers
InvoiceRepository invoiceRepository,
CurrencyNameTable currencyNameTable,
UserManager<ApplicationUser> userManager,
BTCPayRateProviderFactory rateProvider,
RateFetcher rateProvider,
StoreRepository storeRepository,
EventAggregator eventAggregator,
BTCPayWalletProvider walletProvider,

View file

@ -15,12 +15,12 @@ namespace BTCPayServer.Controllers
{
public class RateController : Controller
{
BTCPayRateProviderFactory _RateProviderFactory;
RateFetcher _RateProviderFactory;
BTCPayNetworkProvider _NetworkProvider;
CurrencyNameTable _CurrencyNameTable;
StoreRepository _StoreRepo;
public RateController(
BTCPayRateProviderFactory rateProviderFactory,
RateFetcher rateProviderFactory,
BTCPayNetworkProvider networkProvider,
StoreRepository storeRepo,
CurrencyNameTable currencyNameTable)

View file

@ -33,14 +33,14 @@ namespace BTCPayServer.Controllers
private UserManager<ApplicationUser> _UserManager;
SettingsRepository _SettingsRepository;
private readonly NBXplorerDashboard _dashBoard;
private BTCPayRateProviderFactory _RateProviderFactory;
private RateFetcher _RateProviderFactory;
private StoreRepository _StoreRepository;
LightningConfigurationProvider _LnConfigProvider;
BTCPayServerOptions _Options;
public ServerController(UserManager<ApplicationUser> userManager,
Configuration.BTCPayServerOptions options,
BTCPayRateProviderFactory rateProviderFactory,
RateFetcher rateProviderFactory,
SettingsRepository settingsRepository,
NBXplorerDashboard dashBoard,
LightningConfigurationProvider lnConfigProvider,

View file

@ -35,7 +35,7 @@ namespace BTCPayServer.Controllers
[AutoValidateAntiforgeryToken]
public partial class StoresController : Controller
{
BTCPayRateProviderFactory _RateFactory;
RateFetcher _RateFactory;
public string CreatedStoreId { get; set; }
public StoresController(
IServiceProvider serviceProvider,
@ -47,7 +47,7 @@ namespace BTCPayServer.Controllers
AccessTokenController tokenController,
BTCPayWalletProvider walletProvider,
BTCPayNetworkProvider networkProvider,
BTCPayRateProviderFactory rateFactory,
RateFetcher rateFactory,
ExplorerClientProvider explorerProvider,
IFeeProviderFactory feeRateProvider,
LanguageService langService,
@ -521,7 +521,7 @@ namespace BTCPayServer.Controllers
private CoinAverageExchange[] GetSupportedExchanges()
{
return _RateFactory.GetSupportedExchanges()
return _RateFactory.RateProviderFactory.GetSupportedExchanges()
.Select(c => c.Value)
.OrderBy(s => s.Name, StringComparer.OrdinalIgnoreCase)
.ToArray();

View file

@ -42,7 +42,7 @@ namespace BTCPayServer.Controllers
private readonly ExplorerClientProvider _explorerProvider;
private readonly IFeeProviderFactory _feeRateProvider;
private readonly BTCPayWalletProvider _walletProvider;
BTCPayRateProviderFactory _RateProvider;
RateFetcher _RateProvider;
CurrencyNameTable _currencyTable;
public WalletsController(StoreRepository repo,
CurrencyNameTable currencyTable,
@ -50,7 +50,7 @@ namespace BTCPayServer.Controllers
UserManager<ApplicationUser> userManager,
IOptions<MvcJsonOptions> mvcJsonOptions,
NBXplorerDashboard dashboard,
BTCPayRateProviderFactory rateProvider,
RateFetcher rateProvider,
ExplorerClientProvider explorerProvider,
IFeeProviderFactory feeRateProvider,
BTCPayWalletProvider walletProvider)

View file

@ -18,9 +18,9 @@ namespace BTCPayServer.HostedServices
{
private SettingsRepository _SettingsRepository;
private CoinAverageSettings _coinAverageSettings;
BTCPayRateProviderFactory _RateProviderFactory;
RateProviderFactory _RateProviderFactory;
public RatesHostedService(SettingsRepository repo,
BTCPayRateProviderFactory rateProviderFactory,
RateProviderFactory rateProviderFactory,
CoinAverageSettings coinAverageSettings)
{
this._SettingsRepository = repo;

View file

@ -138,7 +138,8 @@ namespace BTCPayServer.Hosting
else
return new Bitpay(new Key(), new Uri("https://test.bitpay.com/"));
});
services.TryAddSingleton<BTCPayRateProviderFactory>();
services.TryAddSingleton<RateProviderFactory>();
services.TryAddSingleton<RateFetcher>();
services.TryAddScoped<IHttpContextAccessor, HttpContextAccessor>();
services.AddTransient<AccessTokenController>();

View file

@ -0,0 +1,83 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.ExceptionServices;
using System.Threading.Tasks;
using BTCPayServer.Data;
using BTCPayServer.Rating;
namespace BTCPayServer.Services.Rates
{
public class BackgroundFetcherRateProvider : IRateProvider
{
class LatestFetch
{
public ExchangeRates Latest;
public DateTimeOffset Timestamp;
public Exception Exception;
internal ExchangeRates GetResult()
{
if (Exception != null)
{
ExceptionDispatchInfo.Capture(Exception).Throw();
}
return Latest;
}
}
IRateProvider _Inner;
public BackgroundFetcherRateProvider(IRateProvider inner)
{
if (inner == null)
throw new ArgumentNullException(nameof(inner));
_Inner = inner;
}
public TimeSpan RefreshRate { get; set; } = TimeSpan.FromSeconds(30);
public DateTimeOffset NextUpdate
{
get
{
var latest = _Latest;
if (latest == null)
return DateTimeOffset.UtcNow;
return latest.Timestamp + RefreshRate;
}
}
public async Task<bool> UpdateIfNecessary()
{
if (NextUpdate <= DateTimeOffset.UtcNow)
{
await Fetch();
return true;
}
return false;
}
LatestFetch _Latest;
public async Task<ExchangeRates> GetRatesAsync()
{
return (_Latest ?? (await Fetch())).GetResult();
}
private async Task<LatestFetch> Fetch()
{
var fetch = new LatestFetch();
try
{
var rates = await _Inner.GetRatesAsync();
fetch.Latest = rates;
}
catch (Exception ex)
{
fetch.Exception = ex;
}
fetch.Timestamp = DateTimeOffset.UtcNow;
_Latest = fetch;
return fetch;
}
}
}

View file

@ -0,0 +1,95 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using BTCPayServer.Rating;
using ExchangeSharp;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Options;
using static BTCPayServer.Services.Rates.RateProviderFactory;
namespace BTCPayServer.Services.Rates
{
public class ExchangeException
{
public Exception Exception { get; set; }
public string ExchangeName { get; set; }
}
public class RateResult
{
public List<ExchangeException> ExchangeExceptions { get; set; } = new List<ExchangeException>();
public string Rule { get; set; }
public string EvaluatedRule { get; set; }
public HashSet<RateRulesErrors> Errors { get; set; }
public BidAsk BidAsk { get; set; }
public bool Cached { get; internal set; }
}
public class RateFetcher
{
private readonly RateProviderFactory _rateProviderFactory;
public RateFetcher(RateProviderFactory rateProviderFactory)
{
_rateProviderFactory = rateProviderFactory;
}
public RateProviderFactory RateProviderFactory => _rateProviderFactory;
public async Task<RateResult> FetchRate(CurrencyPair pair, RateRules rules)
{
return await FetchRates(new HashSet<CurrencyPair>(new[] { pair }), rules).First().Value;
}
public Dictionary<CurrencyPair, Task<RateResult>> FetchRates(HashSet<CurrencyPair> pairs, RateRules rules)
{
if (rules == null)
throw new ArgumentNullException(nameof(rules));
var fetchingRates = new Dictionary<CurrencyPair, Task<RateResult>>();
var fetchingExchanges = new Dictionary<string, Task<QueryRateResult>>();
var consolidatedRates = new ExchangeRates();
foreach (var i in pairs.Select(p => (Pair: p, RateRule: rules.GetRuleFor(p))))
{
var dependentQueries = new List<Task<QueryRateResult>>();
foreach (var requiredExchange in i.RateRule.ExchangeRates)
{
if (!fetchingExchanges.TryGetValue(requiredExchange.Exchange, out var fetching))
{
fetching = _rateProviderFactory.QueryRates(requiredExchange.Exchange);
fetchingExchanges.Add(requiredExchange.Exchange, fetching);
}
dependentQueries.Add(fetching);
}
fetchingRates.Add(i.Pair, GetRuleValue(dependentQueries, i.RateRule));
}
return fetchingRates;
}
private async Task<RateResult> GetRuleValue(List<Task<QueryRateResult>> dependentQueries, RateRule rateRule)
{
var result = new RateResult();
result.Cached = true;
foreach (var queryAsync in dependentQueries)
{
var query = await queryAsync;
if (!query.CachedResult)
result.Cached = false;
result.ExchangeExceptions.AddRange(query.Exceptions);
foreach (var rule in query.ExchangeRates)
{
rateRule.ExchangeRates.SetRate(rule.Exchange, rule.CurrencyPair, rule.BidAsk);
}
}
rateRule.Reevaluate();
result.BidAsk = rateRule.BidAsk;
result.Errors = rateRule.Errors;
result.EvaluatedRule = rateRule.ToString(true);
result.Rule = rateRule.ToString(false);
return result;
}
}
}

View file

@ -1,5 +1,4 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
@ -11,32 +10,28 @@ using Microsoft.Extensions.Options;
namespace BTCPayServer.Services.Rates
{
public class ExchangeException
public class RateProviderFactory
{
public Exception Exception { get; set; }
public string ExchangeName { get; set; }
}
public class RateResult
{
public List<ExchangeException> ExchangeExceptions { get; set; } = new List<ExchangeException>();
public string Rule { get; set; }
public string EvaluatedRule { get; set; }
public HashSet<RateRulesErrors> Errors { get; set; }
public BidAsk BidAsk { get; set; }
public bool Cached { get; internal set; }
}
public class BTCPayRateProviderFactory
{
class QueryRateResult
public class QueryRateResult
{
public bool CachedResult { get; set; }
public List<ExchangeException> Exceptions { get; set; }
public ExchangeRates ExchangeRates { get; set; }
}
public RateProviderFactory(IOptions<MemoryCacheOptions> cacheOptions,
IHttpClientFactory httpClientFactory,
CoinAverageSettings coinAverageSettings)
{
_httpClientFactory = httpClientFactory;
_CoinAverageSettings = coinAverageSettings;
_Cache = new MemoryCache(cacheOptions);
_CacheOptions = cacheOptions;
// We use 15 min because of limits with free version of bitcoinaverage
CacheSpan = TimeSpan.FromMinutes(15.0);
InitExchanges();
}
IMemoryCache _Cache;
private IOptions<MemoryCacheOptions> _CacheOptions;
private readonly IHttpClientFactory _httpClientFactory;
public IMemoryCache Cache
{
@ -45,25 +40,33 @@ namespace BTCPayServer.Services.Rates
return _Cache;
}
}
CoinAverageSettings _CoinAverageSettings;
public BTCPayRateProviderFactory(IOptions<MemoryCacheOptions> cacheOptions,
IHttpClientFactory httpClientFactory,
BTCPayNetworkProvider btcpayNetworkProvider,
CoinAverageSettings coinAverageSettings)
TimeSpan _CacheSpan;
public TimeSpan CacheSpan
{
if (cacheOptions == null)
throw new ArgumentNullException(nameof(cacheOptions));
_CoinAverageSettings = coinAverageSettings;
_Cache = new MemoryCache(cacheOptions);
_CacheOptions = cacheOptions;
_httpClientFactory = httpClientFactory;
// We use 15 min because of limits with free version of bitcoinaverage
CacheSpan = TimeSpan.FromMinutes(15.0);
this.btcpayNetworkProvider = btcpayNetworkProvider;
InitExchanges();
get
{
return _CacheSpan;
}
set
{
_CacheSpan = value;
InvalidateCache();
}
}
public void InvalidateCache()
{
_Cache = new MemoryCache(_CacheOptions);
}
CoinAverageSettings _CoinAverageSettings;
private readonly IHttpClientFactory _httpClientFactory;
private readonly Dictionary<string, IRateProvider> _DirectProviders = new Dictionary<string, IRateProvider>();
public Dictionary<string, IRateProvider> DirectProviders
{
get
{
return _DirectProviders;
}
}
public bool UseCoinAverageAsFallback { get; set; } = true;
private void InitExchanges()
{
@ -80,7 +83,7 @@ namespace BTCPayServer.Services.Rates
DirectProviders.Add(CoinAverageRateProvider.CoinAverageName, new CoinAverageRateProvider() { Exchange = CoinAverageRateProvider.CoinAverageName, HttpClient = _httpClientFactory?.CreateClient(), Authenticator = _CoinAverageSettings });
// Those exchanges make multiple requests when calling GetTickers so we remove them
DirectProviders.Add("kraken", new KrakenExchangeRateProvider() { HttpClient = _httpClientFactory?.CreateClient() });
DirectProviders.Add("kraken", new KrakenExchangeRateProvider() { HttpClient = _httpClientFactory?.CreateClient() });
//DirectProviders.Add("gdax", new ExchangeSharpRateProvider("gdax", new ExchangeGdaxAPI()));
//DirectProviders.Add("gemini", new ExchangeSharpRateProvider("gemini", new ExchangeGeminiAPI()));
//DirectProviders.Add("bitfinex", new ExchangeSharpRateProvider("bitfinex", new ExchangeBitfinexAPI()));
@ -103,97 +106,13 @@ namespace BTCPayServer.Services.Rates
return exchanges;
}
private readonly Dictionary<string, IRateProvider> _DirectProviders = new Dictionary<string, IRateProvider>();
public Dictionary<string, IRateProvider> DirectProviders
{
get
{
return _DirectProviders;
}
}
BTCPayNetworkProvider btcpayNetworkProvider;
TimeSpan _CacheSpan;
public TimeSpan CacheSpan
{
get
{
return _CacheSpan;
}
set
{
_CacheSpan = value;
InvalidateCache();
}
}
public void InvalidateCache()
{
_Cache = new MemoryCache(_CacheOptions);
}
public async Task<RateResult> FetchRate(CurrencyPair pair, RateRules rules)
{
return await FetchRates(new HashSet<CurrencyPair>(new[] { pair }), rules).First().Value;
}
public Dictionary<CurrencyPair, Task<RateResult>> FetchRates(HashSet<CurrencyPair> pairs, RateRules rules)
{
if (rules == null)
throw new ArgumentNullException(nameof(rules));
var fetchingRates = new Dictionary<CurrencyPair, Task<RateResult>>();
var fetchingExchanges = new Dictionary<string, Task<QueryRateResult>>();
var consolidatedRates = new ExchangeRates();
foreach (var i in pairs.Select(p => (Pair: p, RateRule: rules.GetRuleFor(p))))
{
var dependentQueries = new List<Task<QueryRateResult>>();
foreach (var requiredExchange in i.RateRule.ExchangeRates)
{
if (!fetchingExchanges.TryGetValue(requiredExchange.Exchange, out var fetching))
{
fetching = QueryRates(requiredExchange.Exchange);
fetchingExchanges.Add(requiredExchange.Exchange, fetching);
}
dependentQueries.Add(fetching);
}
fetchingRates.Add(i.Pair, GetRuleValue(dependentQueries, i.RateRule));
}
return fetchingRates;
}
private async Task<RateResult> GetRuleValue(List<Task<QueryRateResult>> dependentQueries, RateRule rateRule)
{
var result = new RateResult();
result.Cached = true;
foreach (var queryAsync in dependentQueries)
{
var query = await queryAsync;
if (!query.CachedResult)
result.Cached = false;
result.ExchangeExceptions.AddRange(query.Exceptions);
foreach (var rule in query.ExchangeRates)
{
rateRule.ExchangeRates.SetRate(rule.Exchange, rule.CurrencyPair, rule.BidAsk);
}
}
rateRule.Reevaluate();
result.BidAsk = rateRule.BidAsk;
result.Errors = rateRule.Errors;
result.EvaluatedRule = rateRule.ToString(true);
result.Rule = rateRule.ToString(false);
return result;
}
private async Task<QueryRateResult> QueryRates(string exchangeName)
public bool UseCoinAverageAsFallback { get; set; } = true;
public async Task<QueryRateResult> QueryRates(string exchangeName)
{
List<IRateProvider> providers = new List<IRateProvider>();
if (DirectProviders.TryGetValue(exchangeName, out var directProvider))
providers.Add(directProvider);
if (_CoinAverageSettings.AvailableExchanges.ContainsKey(exchangeName))
if (UseCoinAverageAsFallback && _CoinAverageSettings.AvailableExchanges.ContainsKey(exchangeName))
{
providers.Add(new CoinAverageRateProvider()
{