From 52f5d21480d9f78ebd8dd4b5b87ee474a8299a51 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Mon, 4 Jul 2022 15:50:56 +0900 Subject: [PATCH] Improve performance of on chain transaction listing for big wallets --- .../Models/TransactionStatus.cs | 3 +- .../StoreRecentTransactions.cs | 65 +++---------- ...GreenfieldStoreOnChainWalletsController.cs | 53 +++++----- .../Controllers/UIWalletsController.cs | 13 +-- BTCPayServer/Services/Wallets/BTCPayWallet.cs | 97 +++++++++++++++++-- .../Services/Wallets/BTCPayWalletProvider.cs | 3 +- .../Wallets/Export/TransactionsExport.cs | 4 +- ...agger.template.stores-wallet.on-chain.json | 6 +- 8 files changed, 136 insertions(+), 108 deletions(-) diff --git a/BTCPayServer.Client/Models/TransactionStatus.cs b/BTCPayServer.Client/Models/TransactionStatus.cs index fc97802f2..1885102d3 100644 --- a/BTCPayServer.Client/Models/TransactionStatus.cs +++ b/BTCPayServer.Client/Models/TransactionStatus.cs @@ -3,7 +3,6 @@ namespace BTCPayServer.Client.Models public enum TransactionStatus { Unconfirmed, - Confirmed, - Replaced + Confirmed } } diff --git a/BTCPayServer/Components/StoreRecentTransactions/StoreRecentTransactions.cs b/BTCPayServer/Components/StoreRecentTransactions/StoreRecentTransactions.cs index 46e50577c..0f29e1f4c 100644 --- a/BTCPayServer/Components/StoreRecentTransactions/StoreRecentTransactions.cs +++ b/BTCPayServer/Components/StoreRecentTransactions/StoreRecentTransactions.cs @@ -27,7 +27,6 @@ public class StoreRecentTransactions : ViewComponent private readonly BTCPayWalletProvider _walletProvider; public BTCPayNetworkProvider NetworkProvider { get; } - public NBXplorerConnectionFactory ConnectionFactory { get; } public StoreRecentTransactions( StoreRepository storeRepo, @@ -38,7 +37,6 @@ public class StoreRecentTransactions : ViewComponent { _storeRepo = storeRepo; NetworkProvider = networkProvider; - ConnectionFactory = connectionFactory; _walletProvider = walletProvider; _dbContextFactory = dbContextFactory; CryptoCode = networkProvider.DefaultNetwork.CryptoCode; @@ -52,57 +50,20 @@ public class StoreRecentTransactions : ViewComponent var transactions = new List(); if (derivationSettings?.AccountDerivation is not null) { - if (ConnectionFactory.Available) - { - var wallet_id = derivationSettings.GetNBXWalletId(); - await using var conn = await ConnectionFactory.OpenConnection(); - var rows = await conn.QueryAsync( - "SELECT t.tx_id, t.seen_at, to_btc(balance_change::NUMERIC) balance_change, (t.blk_id IS NOT NULL) confirmed " + - "FROM get_wallets_recent(@wallet_id, @code, @interval, 5, 0) " + - "JOIN txs t USING (code, tx_id) " + - "ORDER BY seen_at DESC;", - new - { - wallet_id, - code = CryptoCode, - interval = TimeSpan.FromDays(31) - }); - var network = derivationSettings.Network; - foreach (var r in rows) + var network = derivationSettings.Network; + var wallet = _walletProvider.GetWallet(network); + var allTransactions = await wallet.FetchTransactionHistory(derivationSettings.AccountDerivation, 0, 5, TimeSpan.FromDays(31.0)); + transactions = allTransactions + .Select(tx => new StoreRecentTransactionViewModel { - var seenAt = new DateTimeOffset(((DateTime)r.seen_at)); - var balanceChange = new Money((decimal)r.balance_change, MoneyUnit.BTC); - transactions.Add(new StoreRecentTransactionViewModel() - { - Timestamp = seenAt, - Id = r.tx_id, - Balance = balanceChange.ShowMoney(network), - IsConfirmed = r.confirmed, - Link = string.Format(CultureInfo.InvariantCulture, network.BlockExplorerLink, r.tx_id), - Positive = balanceChange.GetValue(network) >= 0, - }); - } - } - else - { - var network = derivationSettings.Network; - var wallet = _walletProvider.GetWallet(network); - var allTransactions = await wallet.FetchTransactions(derivationSettings.AccountDerivation); - transactions = allTransactions.UnconfirmedTransactions.Transactions - .Concat(allTransactions.ConfirmedTransactions.Transactions).ToArray() - .OrderByDescending(t => t.Timestamp) - .Take(5) - .Select(tx => new StoreRecentTransactionViewModel - { - Id = tx.TransactionId.ToString(), - Positive = tx.BalanceChange.GetValue(network) >= 0, - Balance = tx.BalanceChange.ShowMoney(network), - IsConfirmed = tx.Confirmations != 0, - Link = string.Format(CultureInfo.InvariantCulture, network.BlockExplorerLink, tx.TransactionId.ToString()), - Timestamp = tx.Timestamp - }) - .ToList(); - } + Id = tx.TransactionId.ToString(), + Positive = tx.BalanceChange.GetValue(network) >= 0, + Balance = tx.BalanceChange.ShowMoney(network), + IsConfirmed = tx.Confirmations != 0, + Link = string.Format(CultureInfo.InvariantCulture, network.BlockExplorerLink, tx.TransactionId.ToString()), + Timestamp = tx.SeenAt + }) + .ToList(); } diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldStoreOnChainWalletsController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldStoreOnChainWalletsController.cs index f5e91b53c..d8730d432 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldStoreOnChainWalletsController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldStoreOnChainWalletsController.cs @@ -187,41 +187,33 @@ namespace BTCPayServer.Controllers.Greenfield var walletId = new WalletId(storeId, cryptoCode); var walletTransactionsInfoAsync = await _walletRepository.GetWalletTransactionsInfo(walletId); - var txs = await wallet.FetchTransactions(derivationScheme.AccountDerivation); - var filteredFlatList = new List(); - if (statusFilter is null || !statusFilter.Any() || statusFilter.Contains(TransactionStatus.Confirmed)) + var preFiltering = true; + if (statusFilter?.Any() is true || !string.IsNullOrWhiteSpace(labelFilter)) + preFiltering = false; + var txs = await wallet.FetchTransactionHistory(derivationScheme.AccountDerivation, preFiltering ? skip : 0, preFiltering ? limit : int.MaxValue); + if (!preFiltering) { - filteredFlatList.AddRange(txs.ConfirmedTransactions.Transactions); - } - - if (statusFilter is null || !statusFilter.Any() || statusFilter.Contains(TransactionStatus.Unconfirmed)) - { - filteredFlatList.AddRange(txs.UnconfirmedTransactions.Transactions); - } - - if (statusFilter is null || !statusFilter.Any() || statusFilter.Contains(TransactionStatus.Replaced)) - { - filteredFlatList.AddRange(txs.ReplacedTransactions.Transactions); - } - - if (labelFilter != null) - { - filteredFlatList = filteredFlatList.Where(information => + var filteredList = new List(txs.Count); + foreach (var t in txs) { - walletTransactionsInfoAsync.TryGetValue(information.TransactionId.ToString(), out var transactionInfo); - - if (transactionInfo != null) + if (!string.IsNullOrWhiteSpace(labelFilter)) { - return transactionInfo.Labels.ContainsKey(labelFilter); + walletTransactionsInfoAsync.TryGetValue(t.TransactionId.ToString(), out var transactionInfo); + if (transactionInfo?.Labels.ContainsKey(labelFilter) is true) + filteredList.Add(t); } - else + if (statusFilter?.Any() is true) { - return false; + if (statusFilter.Contains(TransactionStatus.Confirmed) && t.Confirmations != 0) + filteredList.Add(t); + else if (statusFilter.Contains(TransactionStatus.Unconfirmed) && t.Confirmations == 0) + filteredList.Add(t); } - }).ToList(); + } + txs = filteredList; } - var result = filteredFlatList.Skip(skip).Take(limit).Select(information => + var result = txs.Skip(skip).Take(limit).Select(information => { walletTransactionsInfoAsync.TryGetValue(information.TransactionId.ToString(), out var transactionInfo); return ToModel(transactionInfo, information, wallet); @@ -671,7 +663,7 @@ namespace BTCPayServer.Controllers.Greenfield } private OnChainWalletTransactionData ToModel(WalletTransactionInfo? walletTransactionsInfoAsync, - TransactionInformation tx, + TransactionHistoryLine tx, BTCPayWallet wallet) { return new OnChainWalletTransactionData() @@ -683,9 +675,8 @@ namespace BTCPayServer.Controllers.Greenfield BlockHash = tx.BlockHash, BlockHeight = tx.Height, Confirmations = tx.Confirmations, - Timestamp = tx.Timestamp, - Status = tx.Confirmations > 0 ? TransactionStatus.Confirmed : - tx.ReplacedBy != null ? TransactionStatus.Replaced : TransactionStatus.Unconfirmed + Timestamp = tx.SeenAt, + Status = tx.Confirmations > 0 ? TransactionStatus.Confirmed : TransactionStatus.Unconfirmed }; } } diff --git a/BTCPayServer/Controllers/UIWalletsController.cs b/BTCPayServer/Controllers/UIWalletsController.cs index eed46829f..2a9e45b92 100644 --- a/BTCPayServer/Controllers/UIWalletsController.cs +++ b/BTCPayServer/Controllers/UIWalletsController.cs @@ -281,7 +281,7 @@ namespace BTCPayServer.Controllers var wallet = _walletProvider.GetWallet(paymentMethod.Network); var walletBlobAsync = WalletRepository.GetWalletInfo(walletId); var walletTransactionsInfoAsync = WalletRepository.GetWalletTransactionsInfo(walletId); - var transactions = await wallet.FetchTransactions(paymentMethod.AccountDerivation); + var transactions = await wallet.FetchTransactionHistory(paymentMethod.AccountDerivation, skip, count); var walletBlob = await walletBlobAsync; var walletTransactionsInfo = await walletTransactionsInfoAsync; var model = new ListTransactionsViewModel { Skip = skip, Count = count }; @@ -301,14 +301,13 @@ namespace BTCPayServer.Controllers } else { - foreach (var tx in transactions.UnconfirmedTransactions.Transactions - .Concat(transactions.ConfirmedTransactions.Transactions).ToArray()) + foreach (var tx in transactions) { var vm = new ListTransactionsViewModel.TransactionViewModel(); vm.Id = tx.TransactionId.ToString(); vm.Link = string.Format(CultureInfo.InvariantCulture, paymentMethod.Network.BlockExplorerLink, vm.Id); - vm.Timestamp = tx.Timestamp; + vm.Timestamp = tx.SeenAt; vm.Positive = tx.BalanceChange.GetValue(wallet.Network) >= 0; vm.Balance = tx.BalanceChange.ShowMoney(wallet.Network); vm.IsConfirmed = tx.Confirmations != 0; @@ -1326,12 +1325,8 @@ namespace BTCPayServer.Controllers var wallet = _walletProvider.GetWallet(paymentMethod.Network); var walletTransactionsInfoAsync = WalletRepository.GetWalletTransactionsInfo(walletId); - var transactions = await wallet.FetchTransactions(paymentMethod.AccountDerivation); + var input = await wallet.FetchTransactionHistory(paymentMethod.AccountDerivation, null, null); var walletTransactionsInfo = await walletTransactionsInfoAsync; - var input = transactions.UnconfirmedTransactions.Transactions - .Concat(transactions.ConfirmedTransactions.Transactions) - .OrderByDescending(t => t.Timestamp) - .ToList(); var export = new TransactionsExport(wallet, walletTransactionsInfo); var res = export.Process(input, format); diff --git a/BTCPayServer/Services/Wallets/BTCPayWallet.cs b/BTCPayServer/Services/Wallets/BTCPayWallet.cs index be8cbfc60..04922570c 100644 --- a/BTCPayServer/Services/Wallets/BTCPayWallet.cs +++ b/BTCPayServer/Services/Wallets/BTCPayWallet.cs @@ -1,4 +1,5 @@ using System; +using Dapper; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; @@ -39,12 +40,13 @@ namespace BTCPayServer.Services.Wallets } public class BTCPayWallet { + public NBXplorerConnectionFactory NbxplorerConnectionFactory { get; } public Logs Logs { get; } private readonly ExplorerClient _Client; private readonly IMemoryCache _MemoryCache; public BTCPayWallet(ExplorerClient client, IMemoryCache memoryCache, BTCPayNetwork network, - ApplicationDbContextFactory dbContextFactory, Logs logs) + ApplicationDbContextFactory dbContextFactory, NBXplorerConnectionFactory nbxplorerConnectionFactory, Logs logs) { ArgumentNullException.ThrowIfNull(client); ArgumentNullException.ThrowIfNull(memoryCache); @@ -52,6 +54,7 @@ namespace BTCPayServer.Services.Wallets _Client = client; _Network = network; _dbContextFactory = dbContextFactory; + NbxplorerConnectionFactory = nbxplorerConnectionFactory; _MemoryCache = memoryCache; } @@ -199,9 +202,81 @@ namespace BTCPayServer.Services.Wallets return await completionSource.Task; } - public async Task FetchTransactions(DerivationStrategyBase derivationStrategyBase) + List dummy = new List(); + public async Task> FetchTransactionHistory(DerivationStrategyBase derivationStrategyBase, int? skip = null, int? count = null, TimeSpan? interval = null) { - return FilterValidTransactions(await _Client.GetTransactionsAsync(derivationStrategyBase)); + // This is two paths: + // * Sometimes we can ask the DB to do the filtering of rows: If that's the case, we should try to filter at the DB level directly as it is the most efficient. + // * Sometimes we can't query the DB or the given network need to do additional filtering. In such case, we can't really filter at the DB level, and we need to fetch all transactions in memory. + var needAdditionalFiltering = _Network.FilterValidTransactions(dummy) != dummy; + if (!NbxplorerConnectionFactory.Available || needAdditionalFiltering) + { + var txs = await FetchTransactions(derivationStrategyBase); + var txinfos = txs.UnconfirmedTransactions.Transactions.Concat(txs.ConfirmedTransactions.Transactions) + .OrderByDescending(t => t.Timestamp) + .Skip(skip is null ? 0 : skip.Value) + .Take(count is null ? int.MaxValue : count.Value); + var lines = new List(Math.Min((count is int v ? v : int.MaxValue), txs.UnconfirmedTransactions.Transactions.Count + txs.ConfirmedTransactions.Transactions.Count)); + DateTimeOffset? timestampLimit = interval is TimeSpan i ? DateTimeOffset.UtcNow - i : null; + foreach (var t in txinfos) + { + if (timestampLimit is DateTimeOffset l && + t.Timestamp <= l) + break; + lines.Add(FromTransactionInformation(t)); + } + return lines; + } + // This call is more efficient for big wallets, as it doesn't need to load all transactions from the history + else + { + await using var ctx = await NbxplorerConnectionFactory.OpenConnection(); + var rows = await ctx.QueryAsync<(string tx_id, DateTimeOffset seen_at, string blk_id, long? blk_height, long balance_change, string asset_id, long confs)>( + "SELECT r.tx_id, r.seen_at, t.blk_id, t.blk_height, r.balance_change, r.asset_id, COALESCE((SELECT height FROM get_tip('BTC')) - t.blk_height + 1, 0) AS confs " + + "FROM get_wallets_recent(@wallet_id, @code, @interval, @count, @skip) r " + + "JOIN txs t USING (code, tx_id) " + + "ORDER BY r.seen_at DESC", new + { + wallet_id = NBXplorer.Client.DBUtils.nbxv1_get_wallet_id(Network.CryptoCode, derivationStrategyBase.ToString()), + code = Network.CryptoCode, + count, + skip, + interval = interval is TimeSpan t ? t : TimeSpan.FromDays(365 * 1000) + }); + rows.TryGetNonEnumeratedCount(out int c); + var lines = new List(c); + foreach (var row in rows) + { + lines.Add(new TransactionHistoryLine() + { + BalanceChange = string.IsNullOrEmpty(row.asset_id) ? Money.Satoshis(row.balance_change) : new AssetMoney(uint256.Parse(row.asset_id), row.balance_change), + Height = row.blk_height, + SeenAt = row.seen_at, + TransactionId = uint256.Parse(row.tx_id), + Confirmations = row.confs, + BlockHash = string.IsNullOrEmpty(row.asset_id) ? null : uint256.Parse(row.blk_id) + }); + } + return lines; + } + } + + private static TransactionHistoryLine FromTransactionInformation(TransactionInformation t) + { + return new TransactionHistoryLine() + { + BalanceChange = t.BalanceChange, + Confirmations = t.Confirmations, + Height = t.Height, + SeenAt = t.Timestamp, + TransactionId = t.TransactionId + }; + } + + private async Task FetchTransactions(DerivationStrategyBase derivationStrategyBase) + { + var transactions = await _Client.GetTransactionsAsync(derivationStrategyBase); + return FilterValidTransactions(transactions); } private GetTransactionsResponse FilterValidTransactions(GetTransactionsResponse response) @@ -226,7 +301,7 @@ namespace BTCPayServer.Services.Wallets }; } - public async Task FetchTransaction(DerivationStrategyBase derivationStrategyBase, uint256 transactionId) + public async Task FetchTransaction(DerivationStrategyBase derivationStrategyBase, uint256 transactionId) { var tx = await _Client.GetTransactionAsync(derivationStrategyBase, transactionId); if (tx is null || !_Network.FilterValidTransactions(new List() { tx }).Any()) @@ -234,7 +309,7 @@ namespace BTCPayServer.Services.Wallets return null; } - return tx; + return FromTransactionInformation(tx); } public Task BroadcastTransactionsAsync(List transactions) @@ -243,8 +318,6 @@ namespace BTCPayServer.Services.Wallets return Task.WhenAll(tasks); } - - public async Task GetUnspentCoins( DerivationStrategyBase derivationStrategy, bool excludeUnconfirmed = false, @@ -276,4 +349,14 @@ namespace BTCPayServer.Services.Wallets }); } } + + public class TransactionHistoryLine + { + public DateTimeOffset SeenAt { get; set; } + public long? Height { get; set; } + public long Confirmations { get; set; } + public uint256 TransactionId { get; set; } + public uint256 BlockHash { get; set; } + public IMoney BalanceChange { get; set; } + } } diff --git a/BTCPayServer/Services/Wallets/BTCPayWalletProvider.cs b/BTCPayServer/Services/Wallets/BTCPayWalletProvider.cs index 194ef1b89..8e03e0bf8 100644 --- a/BTCPayServer/Services/Wallets/BTCPayWalletProvider.cs +++ b/BTCPayServer/Services/Wallets/BTCPayWalletProvider.cs @@ -18,6 +18,7 @@ namespace BTCPayServer.Services.Wallets IOptions memoryCacheOption, Data.ApplicationDbContextFactory dbContextFactory, BTCPayNetworkProvider networkProvider, + NBXplorerConnectionFactory nbxplorerConnectionFactory, Logs logs) { ArgumentNullException.ThrowIfNull(client); @@ -31,7 +32,7 @@ namespace BTCPayServer.Services.Wallets var explorerClient = _Client.GetExplorerClient(network.CryptoCode); if (explorerClient == null) continue; - _Wallets.Add(network.CryptoCode.ToUpperInvariant(), new BTCPayWallet(explorerClient, new MemoryCache(_Options), network, dbContextFactory, Logs)); + _Wallets.Add(network.CryptoCode.ToUpperInvariant(), new BTCPayWallet(explorerClient, new MemoryCache(_Options), network, dbContextFactory, nbxplorerConnectionFactory, Logs)); } } diff --git a/BTCPayServer/Services/Wallets/Export/TransactionsExport.cs b/BTCPayServer/Services/Wallets/Export/TransactionsExport.cs index 84cb2104a..183964f1a 100644 --- a/BTCPayServer/Services/Wallets/Export/TransactionsExport.cs +++ b/BTCPayServer/Services/Wallets/Export/TransactionsExport.cs @@ -26,14 +26,14 @@ namespace BTCPayServer.Services.Wallets.Export _walletTransactionsInfo = walletTransactionsInfo; } - public string Process(IEnumerable inputList, string fileFormat) + public string Process(IEnumerable inputList, string fileFormat) { var list = inputList.Select(tx => { var model = new ExportTransaction { TransactionId = tx.TransactionId.ToString(), - Timestamp = tx.Timestamp, + Timestamp = tx.SeenAt, Amount = tx.BalanceChange.ShowMoney(_wallet.Network), Currency = _wallet.Network.CryptoCode, IsConfirmed = tx.Confirmations != 0 diff --git a/BTCPayServer/wwwroot/swagger/v1/swagger.template.stores-wallet.on-chain.json b/BTCPayServer/wwwroot/swagger/v1/swagger.template.stores-wallet.on-chain.json index 7290f35fa..0c44beed3 100644 --- a/BTCPayServer/wwwroot/swagger/v1/swagger.template.stores-wallet.on-chain.json +++ b/BTCPayServer/wwwroot/swagger/v1/swagger.template.stores-wallet.on-chain.json @@ -689,13 +689,11 @@ "description": "", "x-enumNames": [ "Unconfirmed", - "Confirmed", - "Replaced" + "Confirmed" ], "enum": [ "Unconfirmed", - "Confirmed", - "Replaced" + "Confirmed" ] }, "LabelData": {