mirrors/btcpayserver
1
0
Fork 0
mirror of https://github.com/btcpayserver/btcpayserver.git synced 2024-11-19 09:54:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3926 from NicolasDorier/improveperfbigwallet

Browse Source 
Improve performance of on chain transaction listing for big wallets
This commit is contained in:
Nicolas Dorier 2022-07-05 14:54:27 +09:00 committed by GitHub
commit 8873c51f2b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 146 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
BTCPayServer.Client/Models/TransactionStatus.cs
View File

@ -3,7 +3,6 @@ namespace BTCPayServer.Client.Models
public enum TransactionStatus
{
Unconfirmed,
Confirmed,
Replaced
Confirmed
}
}

7
BTCPayServer.Tests/BTCPayServerTester.cs
View File

@ -75,6 +75,10 @@ namespace BTCPayServer.Tests
{
get; set;
}
public string ExplorerPostgres
{
get; set;
}
IWebHost _Host;
public int Port
@ -154,6 +158,9 @@ namespace BTCPayServer.Tests
config.AppendLine($"mysql=" + MySQL);
else if (!String.IsNullOrEmpty(Postgres))
config.AppendLine($"postgres=" + Postgres);
if (!string.IsNullOrEmpty(ExplorerPostgres))
config.AppendLine($"explorer.postgres=" + ExplorerPostgres);
var confPath = Path.Combine(chainDirectory, "settings.config");
await File.WriteAllTextAsync(confPath, config.ToString());

1
BTCPayServer.Tests/ServerTester.cs
View File

@ -50,6 +50,7 @@ namespace BTCPayServer.Tests
// since in dev we already can have some users / stores registered, while on CI database is being initalized
// for the first time and first registered user gets admin status by default
Postgres = GetEnvironment("TESTS_POSTGRES", "User ID=postgres;Include Error Detail=true;Host=127.0.0.1;Port=39372;Database=btcpayserver"),
ExplorerPostgres = GetEnvironment("TESTS_EXPLORER_POSTGRES", "User ID=postgres;Include Error Detail=true;Host=127.0.0.1;Port=39372;Database=nbxplorer"),
MySQL = GetEnvironment("TESTS_MYSQL", "User ID=root;Host=127.0.0.1;Port=33036;Database=btcpayserver")
};
if (newDb)

1
BTCPayServer.Tests/docker-compose.altcoins.yml
View File

@ -18,6 +18,7 @@ services:
TESTS_LTCNBXPLORERURL: http://nbxplorer:32838/
TESTS_DB: "Postgres"
TESTS_POSTGRES: User ID=postgres;Include Error Detail=true;Host=postgres;Port=5432;Database=btcpayserver
TESTS_EXPLORER_POSTGRES: User ID=postgres;Include Error Detail=true;Host=postgres;Port=5432;Database=nbxplorer
TESTS_HOSTNAME: tests
TESTS_RUN_EXTERNAL_INTEGRATION: ${TESTS_RUN_EXTERNAL_INTEGRATION:-"false"}
TESTS_AzureBlobStorageConnectionString: ${TESTS_AzureBlobStorageConnectionString:-none}

1
BTCPayServer.Tests/docker-compose.yml
View File

@ -16,6 +16,7 @@ services:
TESTS_BTCNBXPLORERURL: http://nbxplorer:32838/
TESTS_DB: "Postgres"
TESTS_POSTGRES: User ID=postgres;Include Error Detail=true;Host=postgres;Port=5432;Database=btcpayserver
TESTS_EXPLORER_POSTGRES: User ID=postgres;Include Error Detail=true;Host=postgres;Port=5432;Database=nbxplorer
TESTS_HOSTNAME: tests
TESTS_RUN_EXTERNAL_INTEGRATION: ${TESTS_RUN_EXTERNAL_INTEGRATION:-"false"}
TESTS_AzureBlobStorageConnectionString: ${TESTS_AzureBlobStorageConnectionString:-none}

65
BTCPayServer/Components/StoreRecentTransactions/StoreRecentTransactions.cs
View File

@ -27,7 +27,6 @@ public class StoreRecentTransactions : ViewComponent
private readonly BTCPayWalletProvider _walletProvider;
public BTCPayNetworkProvider NetworkProvider { get; }
public NBXplorerConnectionFactory ConnectionFactory { get; }
public StoreRecentTransactions(
StoreRepository storeRepo,
@ -38,7 +37,6 @@ public class StoreRecentTransactions : ViewComponent
{
_storeRepo = storeRepo;
NetworkProvider = networkProvider;
ConnectionFactory = connectionFactory;
_walletProvider = walletProvider;
_dbContextFactory = dbContextFactory;
CryptoCode = networkProvider.DefaultNetwork.CryptoCode;
@ -52,57 +50,20 @@ public class StoreRecentTransactions : ViewComponent
var transactions = new List<StoreRecentTransactionViewModel>();
if (derivationSettings?.AccountDerivation is not null)
{
if (ConnectionFactory.Available)
{
var wallet_id = derivationSettings.GetNBXWalletId();
await using var conn = await ConnectionFactory.OpenConnection();
var rows = await conn.QueryAsync(
"SELECT t.tx_id, t.seen_at, to_btc(balance_change::NUMERIC) balance_change, (t.blk_id IS NOT NULL) confirmed " +
"FROM get_wallets_recent(@wallet_id, @code, @interval, 5, 0) " +
"JOIN txs t USING (code, tx_id) " +
"ORDER BY seen_at DESC;",
new
{
wallet_id,
code = CryptoCode,
interval = TimeSpan.FromDays(31)
});
var network = derivationSettings.Network;
foreach (var r in rows)
var network = derivationSettings.Network;
var wallet = _walletProvider.GetWallet(network);
var allTransactions = await wallet.FetchTransactionHistory(derivationSettings.AccountDerivation, 0, 5, TimeSpan.FromDays(31.0));
transactions = allTransactions
.Select(tx => new StoreRecentTransactionViewModel
{
var seenAt = new DateTimeOffset(((DateTime)r.seen_at));
var balanceChange = new Money((decimal)r.balance_change, MoneyUnit.BTC);
transactions.Add(new StoreRecentTransactionViewModel()
{
Timestamp = seenAt,
Id = r.tx_id,
Balance = balanceChange.ShowMoney(network),
IsConfirmed = r.confirmed,
Link = string.Format(CultureInfo.InvariantCulture, network.BlockExplorerLink, r.tx_id),
Positive = balanceChange.GetValue(network) >= 0,
});
}
}
else
{
var network = derivationSettings.Network;
var wallet = _walletProvider.GetWallet(network);
var allTransactions = await wallet.FetchTransactions(derivationSettings.AccountDerivation);
transactions = allTransactions.UnconfirmedTransactions.Transactions
.Concat(allTransactions.ConfirmedTransactions.Transactions).ToArray()
.OrderByDescending(t => t.Timestamp)
.Take(5)
.Select(tx => new StoreRecentTransactionViewModel
{
Id = tx.TransactionId.ToString(),
Positive = tx.BalanceChange.GetValue(network) >= 0,
Balance = tx.BalanceChange.ShowMoney(network),
IsConfirmed = tx.Confirmations != 0,
Link = string.Format(CultureInfo.InvariantCulture, network.BlockExplorerLink, tx.TransactionId.ToString()),
Timestamp = tx.Timestamp
})
.ToList();
}
Id = tx.TransactionId.ToString(),
Positive = tx.BalanceChange.GetValue(network) >= 0,
Balance = tx.BalanceChange.ShowMoney(network),
IsConfirmed = tx.Confirmations != 0,
Link = string.Format(CultureInfo.InvariantCulture, network.BlockExplorerLink, tx.TransactionId.ToString()),
Timestamp = tx.SeenAt
})
.ToList();
}

53
BTCPayServer/Controllers/GreenField/GreenfieldStoreOnChainWalletsController.cs
View File

@ -187,41 +187,33 @@ namespace BTCPayServer.Controllers.Greenfield
var walletId = new WalletId(storeId, cryptoCode);
var walletTransactionsInfoAsync = await _walletRepository.GetWalletTransactionsInfo(walletId);
var txs = await wallet.FetchTransactions(derivationScheme.AccountDerivation);
var filteredFlatList = new List<TransactionInformation>();
if (statusFilter is null || !statusFilter.Any() || statusFilter.Contains(TransactionStatus.Confirmed))
var preFiltering = true;
if (statusFilter?.Any() is true || !string.IsNullOrWhiteSpace(labelFilter))
preFiltering = false;
var txs = await wallet.FetchTransactionHistory(derivationScheme.AccountDerivation, preFiltering ? skip : 0, preFiltering ? limit : int.MaxValue);
if (!preFiltering)
{
filteredFlatList.AddRange(txs.ConfirmedTransactions.Transactions);
}
if (statusFilter is null || !statusFilter.Any() || statusFilter.Contains(TransactionStatus.Unconfirmed))
{
filteredFlatList.AddRange(txs.UnconfirmedTransactions.Transactions);
}
if (statusFilter is null || !statusFilter.Any() || statusFilter.Contains(TransactionStatus.Replaced))
{
filteredFlatList.AddRange(txs.ReplacedTransactions.Transactions);
}
if (labelFilter != null)
{
filteredFlatList = filteredFlatList.Where(information =>
var filteredList = new List<TransactionHistoryLine>(txs.Count);
foreach (var t in txs)
{
walletTransactionsInfoAsync.TryGetValue(information.TransactionId.ToString(), out var transactionInfo);
if (transactionInfo != null)
if (!string.IsNullOrWhiteSpace(labelFilter))
{
return transactionInfo.Labels.ContainsKey(labelFilter);
walletTransactionsInfoAsync.TryGetValue(t.TransactionId.ToString(), out var transactionInfo);
if (transactionInfo?.Labels.ContainsKey(labelFilter) is true)
filteredList.Add(t);
}
else
if (statusFilter?.Any() is true)
{
return false;
if (statusFilter.Contains(TransactionStatus.Confirmed) && t.Confirmations != 0)
filteredList.Add(t);
else if (statusFilter.Contains(TransactionStatus.Unconfirmed) && t.Confirmations == 0)
filteredList.Add(t);
}
}).ToList();
}
txs = filteredList;
}
var result = filteredFlatList.Skip(skip).Take(limit).Select(information =>
var result = txs.Skip(skip).Take(limit).Select(information =>
{
walletTransactionsInfoAsync.TryGetValue(information.TransactionId.ToString(), out var transactionInfo);
return ToModel(transactionInfo, information, wallet);
@ -671,7 +663,7 @@ namespace BTCPayServer.Controllers.Greenfield
}
private OnChainWalletTransactionData ToModel(WalletTransactionInfo? walletTransactionsInfoAsync,
TransactionInformation tx,
TransactionHistoryLine tx,
BTCPayWallet wallet)
{
return new OnChainWalletTransactionData()
@ -683,9 +675,8 @@ namespace BTCPayServer.Controllers.Greenfield
BlockHash = tx.BlockHash,
BlockHeight = tx.Height,
Confirmations = tx.Confirmations,
Timestamp = tx.Timestamp,
Status = tx.Confirmations > 0 ? TransactionStatus.Confirmed :
tx.ReplacedBy != null ? TransactionStatus.Replaced : TransactionStatus.Unconfirmed
Timestamp = tx.SeenAt,
Status = tx.Confirmations > 0 ? TransactionStatus.Confirmed : TransactionStatus.Unconfirmed
};
}
}

13
BTCPayServer/Controllers/UIWalletsController.cs
View File

@ -281,7 +281,7 @@ namespace BTCPayServer.Controllers
var wallet = _walletProvider.GetWallet(paymentMethod.Network);
var walletBlobAsync = WalletRepository.GetWalletInfo(walletId);
var walletTransactionsInfoAsync = WalletRepository.GetWalletTransactionsInfo(walletId);
var transactions = await wallet.FetchTransactions(paymentMethod.AccountDerivation);
var transactions = await wallet.FetchTransactionHistory(paymentMethod.AccountDerivation, skip, count);
var walletBlob = await walletBlobAsync;
var walletTransactionsInfo = await walletTransactionsInfoAsync;
var model = new ListTransactionsViewModel { Skip = skip, Count = count };
@ -301,14 +301,13 @@ namespace BTCPayServer.Controllers
}
else
{
foreach (var tx in transactions.UnconfirmedTransactions.Transactions
.Concat(transactions.ConfirmedTransactions.Transactions).ToArray())
foreach (var tx in transactions)
{
var vm = new ListTransactionsViewModel.TransactionViewModel();
vm.Id = tx.TransactionId.ToString();
vm.Link = string.Format(CultureInfo.InvariantCulture, paymentMethod.Network.BlockExplorerLink,
vm.Id);
vm.Timestamp = tx.Timestamp;
vm.Timestamp = tx.SeenAt;
vm.Positive = tx.BalanceChange.GetValue(wallet.Network) >= 0;
vm.Balance = tx.BalanceChange.ShowMoney(wallet.Network);
vm.IsConfirmed = tx.Confirmations != 0;
@ -1326,12 +1325,8 @@ namespace BTCPayServer.Controllers
var wallet = _walletProvider.GetWallet(paymentMethod.Network);
var walletTransactionsInfoAsync = WalletRepository.GetWalletTransactionsInfo(walletId);
var transactions = await wallet.FetchTransactions(paymentMethod.AccountDerivation);
var input = await wallet.FetchTransactionHistory(paymentMethod.AccountDerivation, null, null);
var walletTransactionsInfo = await walletTransactionsInfoAsync;
var input = transactions.UnconfirmedTransactions.Transactions
.Concat(transactions.ConfirmedTransactions.Transactions)
.OrderByDescending(t => t.Timestamp)
.ToList();
var export = new TransactionsExport(wallet, walletTransactionsInfo);
var res = export.Process(input, format);

97
BTCPayServer/Services/Wallets/BTCPayWallet.cs
View File

@ -1,4 +1,5 @@
using System;
using Dapper;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
@ -39,12 +40,13 @@ namespace BTCPayServer.Services.Wallets
}
public class BTCPayWallet
{
public NBXplorerConnectionFactory NbxplorerConnectionFactory { get; }
public Logs Logs { get; }
private readonly ExplorerClient _Client;
private readonly IMemoryCache _MemoryCache;
public BTCPayWallet(ExplorerClient client, IMemoryCache memoryCache, BTCPayNetwork network,
ApplicationDbContextFactory dbContextFactory, Logs logs)
ApplicationDbContextFactory dbContextFactory, NBXplorerConnectionFactory nbxplorerConnectionFactory, Logs logs)
{
ArgumentNullException.ThrowIfNull(client);
ArgumentNullException.ThrowIfNull(memoryCache);
@ -52,6 +54,7 @@ namespace BTCPayServer.Services.Wallets
_Client = client;
_Network = network;
_dbContextFactory = dbContextFactory;
NbxplorerConnectionFactory = nbxplorerConnectionFactory;
_MemoryCache = memoryCache;
}
@ -199,9 +202,81 @@ namespace BTCPayServer.Services.Wallets
return await completionSource.Task;
}
public async Task<GetTransactionsResponse> FetchTransactions(DerivationStrategyBase derivationStrategyBase)
List<TransactionInformation> dummy = new List<TransactionInformation>();
public async Task<IList<TransactionHistoryLine>> FetchTransactionHistory(DerivationStrategyBase derivationStrategyBase, int? skip = null, int? count = null, TimeSpan? interval = null)
{
return FilterValidTransactions(await _Client.GetTransactionsAsync(derivationStrategyBase));
// This is two paths:
// * Sometimes we can ask the DB to do the filtering of rows: If that's the case, we should try to filter at the DB level directly as it is the most efficient.
// * Sometimes we can't query the DB or the given network need to do additional filtering. In such case, we can't really filter at the DB level, and we need to fetch all transactions in memory.
var needAdditionalFiltering = _Network.FilterValidTransactions(dummy) != dummy;
if (!NbxplorerConnectionFactory.Available || needAdditionalFiltering)
{
var txs = await FetchTransactions(derivationStrategyBase);
var txinfos = txs.UnconfirmedTransactions.Transactions.Concat(txs.ConfirmedTransactions.Transactions)
.OrderByDescending(t => t.Timestamp)
.Skip(skip is null ? 0 : skip.Value)
.Take(count is null ? int.MaxValue : count.Value);
var lines = new List<TransactionHistoryLine>(Math.Min((count is int v ? v : int.MaxValue), txs.UnconfirmedTransactions.Transactions.Count + txs.ConfirmedTransactions.Transactions.Count));
DateTimeOffset? timestampLimit = interval is TimeSpan i ? DateTimeOffset.UtcNow - i : null;
foreach (var t in txinfos)
{
if (timestampLimit is DateTimeOffset l &&
t.Timestamp <= l)
break;
lines.Add(FromTransactionInformation(t));
}
return lines;
}
// This call is more efficient for big wallets, as it doesn't need to load all transactions from the history
else
{
await using var ctx = await NbxplorerConnectionFactory.OpenConnection();
var rows = await ctx.QueryAsync<(string tx_id, DateTimeOffset seen_at, string blk_id, long? blk_height, long balance_change, string asset_id, long confs)>(
"SELECT r.tx_id, r.seen_at, t.blk_id, t.blk_height, r.balance_change, r.asset_id, COALESCE((SELECT height FROM get_tip('BTC')) - t.blk_height + 1, 0) AS confs " +
"FROM get_wallets_recent(@wallet_id, @code, @interval, @count, @skip) r " +
"JOIN txs t USING (code, tx_id) " +
"ORDER BY r.seen_at DESC", new
{
wallet_id = NBXplorer.Client.DBUtils.nbxv1_get_wallet_id(Network.CryptoCode, derivationStrategyBase.ToString()),
code = Network.CryptoCode,
count,
skip,
interval = interval is TimeSpan t ? t : TimeSpan.FromDays(365 * 1000)
});
rows.TryGetNonEnumeratedCount(out int c);
var lines = new List<TransactionHistoryLine>(c);
foreach (var row in rows)
{
lines.Add(new TransactionHistoryLine()
{
BalanceChange = string.IsNullOrEmpty(row.asset_id) ? Money.Satoshis(row.balance_change) : new AssetMoney(uint256.Parse(row.asset_id), row.balance_change),
Height = row.blk_height,
SeenAt = row.seen_at,
TransactionId = uint256.Parse(row.tx_id),
Confirmations = row.confs,
BlockHash = string.IsNullOrEmpty(row.asset_id) ? null : uint256.Parse(row.blk_id)
});
}
return lines;
}
}
private static TransactionHistoryLine FromTransactionInformation(TransactionInformation t)
{
return new TransactionHistoryLine()
{
BalanceChange = t.BalanceChange,
Confirmations = t.Confirmations,
Height = t.Height,
SeenAt = t.Timestamp,
TransactionId = t.TransactionId
};
}
private async Task<GetTransactionsResponse> FetchTransactions(DerivationStrategyBase derivationStrategyBase)
{
var transactions = await _Client.GetTransactionsAsync(derivationStrategyBase);
return FilterValidTransactions(transactions);
}
private GetTransactionsResponse FilterValidTransactions(GetTransactionsResponse response)
@ -226,7 +301,7 @@ namespace BTCPayServer.Services.Wallets
};
}
public async Task<TransactionInformation> FetchTransaction(DerivationStrategyBase derivationStrategyBase, uint256 transactionId)
public async Task<TransactionHistoryLine> FetchTransaction(DerivationStrategyBase derivationStrategyBase, uint256 transactionId)
{
var tx = await _Client.GetTransactionAsync(derivationStrategyBase, transactionId);
if (tx is null || !_Network.FilterValidTransactions(new List<TransactionInformation>() { tx }).Any())
@ -234,7 +309,7 @@ namespace BTCPayServer.Services.Wallets
return null;
}
return tx;
return FromTransactionInformation(tx);
}
public Task<BroadcastResult[]> BroadcastTransactionsAsync(List<Transaction> transactions)
@ -243,8 +318,6 @@ namespace BTCPayServer.Services.Wallets
return Task.WhenAll(tasks);
}
public async Task<ReceivedCoin[]> GetUnspentCoins(
DerivationStrategyBase derivationStrategy,
bool excludeUnconfirmed = false,
@ -276,4 +349,14 @@ namespace BTCPayServer.Services.Wallets
});
}
}
public class TransactionHistoryLine
{
public DateTimeOffset SeenAt { get; set; }
public long? Height { get; set; }
public long Confirmations { get; set; }
public uint256 TransactionId { get; set; }
public uint256 BlockHash { get; set; }
public IMoney BalanceChange { get; set; }
}
}

3
BTCPayServer/Services/Wallets/BTCPayWalletProvider.cs
View File

@ -18,6 +18,7 @@ namespace BTCPayServer.Services.Wallets
IOptions<MemoryCacheOptions> memoryCacheOption,
Data.ApplicationDbContextFactory dbContextFactory,
BTCPayNetworkProvider networkProvider,
NBXplorerConnectionFactory nbxplorerConnectionFactory,
Logs logs)
{
ArgumentNullException.ThrowIfNull(client);
@ -31,7 +32,7 @@ namespace BTCPayServer.Services.Wallets
var explorerClient = _Client.GetExplorerClient(network.CryptoCode);
if (explorerClient == null)
continue;
_Wallets.Add(network.CryptoCode.ToUpperInvariant(), new BTCPayWallet(explorerClient, new MemoryCache(_Options), network, dbContextFactory, Logs));
_Wallets.Add(network.CryptoCode.ToUpperInvariant(), new BTCPayWallet(explorerClient, new MemoryCache(_Options), network, dbContextFactory, nbxplorerConnectionFactory, Logs));
}
}

4
BTCPayServer/Services/Wallets/Export/TransactionsExport.cs
View File

@ -26,14 +26,14 @@ namespace BTCPayServer.Services.Wallets.Export
_walletTransactionsInfo = walletTransactionsInfo;
}
public string Process(IEnumerable<TransactionInformation> inputList, string fileFormat)
public string Process(IEnumerable<TransactionHistoryLine> inputList, string fileFormat)
{
var list = inputList.Select(tx =>
{
var model = new ExportTransaction
{
TransactionId = tx.TransactionId.ToString(),
Timestamp = tx.Timestamp,
Timestamp = tx.SeenAt,
Amount = tx.BalanceChange.ShowMoney(_wallet.Network),
Currency = _wallet.Network.CryptoCode,
IsConfirmed = tx.Confirmations != 0

6
BTCPayServer/wwwroot/swagger/v1/swagger.template.stores-wallet.on-chain.json
View File

@ -689,13 +689,11 @@
"description": "",
"x-enumNames": [
"Unconfirmed",
"Confirmed",
"Replaced"
"Confirmed"
],
"enum": [
"Unconfirmed",
"Confirmed",
"Replaced"
"Confirmed"
]
},
"LabelData": {