Adding transaction caching

BTCPayServer.Tests/docker-compose.yml

@@ -39,7 +39,7 @@ services:
       - postgres
 
   bitcoin-nbxplorer:
-    image: nicolasdorier/nbxplorer:1.0.0.45
+    image: nicolasdorier/nbxplorer:1.0.0.47
     ports:
       - "32838:32838"
     expose: 
@@ -57,7 +57,7 @@ services:
       - bitcoind
 
   litecoin-nbxplorer:
-    image: nicolasdorier/nbxplorer:1.0.0.45
+    image: nicolasdorier/nbxplorer:1.0.0.47
     ports:
       - "32839:32839"
     expose: 

BTCPayServer/BTCPayServer.csproj

@@ -24,7 +24,7 @@
     <PackageReference Include="NBitcoin" Version="4.0.0.51" />
     <PackageReference Include="NBitpayClient" Version="1.0.0.16" />	  
     <PackageReference Include="DBreeze" Version="1.87.0" />
-    <PackageReference Include="NBXplorer.Client" Version="1.0.0.28" />
+    <PackageReference Include="NBXplorer.Client" Version="1.0.0.29" />
     <PackageReference Include="NicolasDorier.CommandLine" Version="1.0.0.1" />
     <PackageReference Include="NicolasDorier.CommandLine.Configuration" Version="1.0.0.2" />
     <PackageReference Include="NicolasDorier.StandardConfiguration" Version="1.0.0.13" />

BTCPayServer/HostedServices/InvoiceWatcher.cs

@@ -17,6 +17,7 @@ using BTCPayServer.Controllers;
 using BTCPayServer.Events;
 using Microsoft.AspNetCore.Hosting;
 using BTCPayServer.Services.Invoices;
+using BTCPayServer.Services;
 
 namespace BTCPayServer.HostedServices
 {
@@ -150,7 +151,7 @@ namespace BTCPayServer.HostedServices
             }
 
             var derivationStrategies = invoice.GetDerivationStrategies(_NetworkProvider).ToArray();
-            var payments = await GetPaymentsWithTransaction(null, derivationStrategies, invoice);
+            var payments = await GetPaymentsWithTransaction(derivationStrategies, invoice);
             foreach (Task<NetworkCoins> coinsAsync in GetCoinsPerNetwork(context, invoice, derivationStrategies))
             {
                 var coins = await coinsAsync;
@@ -173,7 +174,7 @@ namespace BTCPayServer.HostedServices
                 }
                 if (dirtyAddress)
                 {
-                    payments = await GetPaymentsWithTransaction(payments, derivationStrategies, invoice);
+                    payments = await GetPaymentsWithTransaction(derivationStrategies, invoice);
                 }
                 var network = coins.Wallet.Network;
                 var cryptoData = invoice.GetCryptoData(network, _NetworkProvider);
@@ -293,58 +294,23 @@ namespace BTCPayServer.HostedServices
         }
 
 
-        class AccountedPaymentEntities : List<AccountedPaymentEntity>
+        private async Task<IEnumerable<AccountedPaymentEntity>> GetPaymentsWithTransaction(DerivationStrategy[] derivations, InvoiceEntity invoice)
-        {
-            public AccountedPaymentEntities(AccountedPaymentEntities existing)
-            {
-                if (existing != null)
-                    _Transactions = existing._Transactions;
-            }
-
-            Dictionary<uint256, TransactionResult> _Transactions = new Dictionary<uint256, TransactionResult>();
-
-            public void AddToCache(IEnumerable<TransactionResult> transactions)
-            {
-                foreach (var tx in transactions)
-                    _Transactions.TryAdd(tx.Transaction.GetHash(), tx);
-            }
-            public TransactionResult GetTransaction(uint256 txId)
-            {
-                _Transactions.TryGetValue(txId, out TransactionResult result);
-                return result;
-            }
-
-            internal IEnumerable<TransactionResult> GetTransactions()
-            {
-                return _Transactions.Values;
-            }
-        }
-        private async Task<AccountedPaymentEntities> GetPaymentsWithTransaction(AccountedPaymentEntities previous, DerivationStrategy[] derivations, InvoiceEntity invoice)
         {
             List<PaymentEntity> updatedPaymentEntities = new List<PaymentEntity>();
-            AccountedPaymentEntities accountedPayments = new AccountedPaymentEntities(previous);
+            List<AccountedPaymentEntity> accountedPayments = new List<AccountedPaymentEntity>();
             foreach (var network in derivations.Select(d => d.Network))
             {
                 var wallet = _WalletProvider.GetWallet(network);
                 if (wallet == null)
                     continue;
 
-                var hashesToFetch = new HashSet<uint256>(invoice
+                var transactions = await wallet.GetTransactions(invoice.GetPayments(wallet.Network)
-                    .GetPayments(network)
                     .Select(t => t.Outpoint.Hash)
-                    .Where(h => accountedPayments?.GetTransaction(h) == null)
+                    .ToArray());
-                    .ToList());
+                var conflicts = GetConflicts(transactions.Select(t => t.Value));
-
-
-                if (hashesToFetch.Count > 0)
-                {
-                    accountedPayments.AddToCache((await wallet.GetTransactions(hashesToFetch.ToArray())).Select(t => t.Value));
-                }
-                var conflicts = GetConflicts(accountedPayments.GetTransactions());
                 foreach (var payment in invoice.GetPayments(network))
                 {
-                    TransactionResult tx = accountedPayments.GetTransaction(payment.Outpoint.Hash);
+                    if (!transactions.TryGetValue(payment.Outpoint.Hash, out TransactionResult tx))
-                    if (tx == null)
                         continue;
 
                     AccountedPaymentEntity accountedPayment = new AccountedPaymentEntity()
@@ -370,7 +336,6 @@ namespace BTCPayServer.HostedServices
             return accountedPayments;
         }
 
-
         class TransactionConflict
         {
             public Dictionary<uint256, TransactionResult> Transactions { get; set; } = new Dictionary<uint256, TransactionResult>();

BTCPayServer/HostedServices/NBXplorerListener.cs

@@ -12,6 +12,7 @@ using NBXplorer;
 using System.Collections.Concurrent;
 using NBXplorer.DerivationStrategy;
 using BTCPayServer.Events;
+using BTCPayServer.Services;
 
 namespace BTCPayServer.HostedServices
 {
@@ -24,9 +25,11 @@ namespace BTCPayServer.HostedServices
         private TaskCompletionSource<bool> _RunningTask;
         private CancellationTokenSource _Cts;
         NBXplorerDashboard _Dashboards;
+        TransactionCacheProvider _TxCache;
 
         public NBXplorerListener(ExplorerClientProvider explorerClients,
                                 NBXplorerDashboard dashboard,
+                                TransactionCacheProvider cacheProvider,
                                 InvoiceRepository invoiceRepository,
                                 EventAggregator aggregator, IApplicationLifetime lifetime)
         {
@@ -36,6 +39,7 @@ namespace BTCPayServer.HostedServices
             _ExplorerClients = explorerClients;
             _Aggregator = aggregator;
             _Lifetime = lifetime;
+            _TxCache = cacheProvider;
         }
 
         CompositeDisposable leases = new CompositeDisposable();
@@ -130,11 +134,13 @@ namespace BTCPayServer.HostedServices
                         switch (newEvent)
                         {
                             case NBXplorer.Models.NewBlockEvent evt:
+                                _TxCache.GetTransactionCache(network).NewBlock(evt.Hash, evt.PreviousBlockHash);
                                 _Aggregator.Publish(new Events.NewBlockEvent());
                                 break;
                             case NBXplorer.Models.NewTransactionEvent evt:
-                                foreach (var txout in evt.Match.Outputs)
+                                foreach (var txout in evt.Outputs)
                                 {
+                                    _TxCache.GetTransactionCache(network).AddToCache(evt.TransactionData);
                                     _Aggregator.Publish(new Events.TxOutReceivedEvent()
                                     {
                                         Network = network,

BTCPayServer/Hosting/BTCPayServerServices.cs

@@ -142,6 +142,8 @@ namespace BTCPayServer.Hosting
                 BlockTarget = 20
             });
 
+            services.AddSingleton<TransactionCacheProvider>();
+
             services.AddSingleton<IHostedService, NBXplorerWaiters>();
             services.AddSingleton<IHostedService, NBXplorerListener>();
             services.AddSingleton<IHostedService, InvoiceNotificationManager>();

BTCPayServer/Services/TransactionCache.cs

@@ -0,0 +1,92 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Options;
+using NBitcoin;
+using NBXplorer.Models;
+
+namespace BTCPayServer.Services
+{
+    public class TransactionCacheProvider
+    {
+        IOptions<MemoryCacheOptions> _Options;
+        public TransactionCacheProvider(IOptions<MemoryCacheOptions> options)
+        {
+            _Options = options;
+        }
+
+        ConcurrentDictionary<string, TransactionCache> _TransactionCaches = new ConcurrentDictionary<string, TransactionCache>();
+        public TransactionCache GetTransactionCache(BTCPayNetwork network)
+        {
+            if (network == null)
+                throw new ArgumentNullException(nameof(network));
+            return _TransactionCaches.GetOrAdd(network.CryptoCode, c => new TransactionCache(_Options, network));
+        }
+    }
+    public class TransactionCache : IDisposable
+    {
+        IOptions<MemoryCacheOptions> _Options;
+        public TransactionCache(IOptions<MemoryCacheOptions> options, BTCPayNetwork network)
+        {
+            if (network == null)
+                throw new ArgumentNullException(nameof(network));
+            _Options = options;
+            _MemoryCache = new MemoryCache(_Options);
+            Network = network;
+        }
+
+        uint256 _LastHash;
+        int _ConfOffset;
+        IMemoryCache _MemoryCache;
+        
+        public void NewBlock(uint256 newHash, uint256 previousHash)
+        {
+            if (_LastHash != previousHash)
+            {
+                var old = _MemoryCache;
+                _ConfOffset = 0;
+                _MemoryCache = new MemoryCache(_Options);
+                Thread.MemoryBarrier();
+                old.Dispose();
+            }
+            else
+                _ConfOffset++;
+            _LastHash = newHash;
+        }
+
+        public TimeSpan CacheSpan { get; private set; } = TimeSpan.FromMinutes(60);
+
+        public BTCPayNetwork Network { get; private set; }
+
+        public void AddToCache(TransactionResult tx)
+        {
+            _MemoryCache.Set(tx.Transaction.GetHash(), tx, DateTimeOffset.UtcNow + CacheSpan);
+        }
+
+
+        public TransactionResult GetTransaction(uint256 txId)
+        {
+            _MemoryCache.TryGetValue(txId.ToString(), out object tx);
+
+            var result = tx as TransactionResult;
+            var confOffset = _ConfOffset;
+            if (result != null && result.Confirmations > 0 && confOffset > 0)
+            {
+                var serializer = new NBXplorer.Serializer(Network.NBitcoinNetwork);
+                result = serializer.ToObject<TransactionResult>(serializer.ToString(result));
+                result.Confirmations += confOffset;
+                result.Height += confOffset;
+            }
+            return result;
+        }
+
+        public void Dispose()
+        {
+            _MemoryCache.Dispose();
+        }
+    }
+}

BTCPayServer/Services/Wallets/BTCPayWallet.cs

@@ -33,13 +33,14 @@ namespace BTCPayServer.Services.Wallets
     public class BTCPayWallet
     {
         private ExplorerClient _Client;
-
+        private TransactionCache _Cache;
-        public BTCPayWallet(ExplorerClient client, BTCPayNetwork network)
+        public BTCPayWallet(ExplorerClient client, TransactionCache cache, BTCPayNetwork network)
         {
             if (client == null)
                 throw new ArgumentNullException(nameof(client));
             _Client = client;
             _Network = network;
+            _Cache = cache;
         }
 
 
@@ -65,11 +66,16 @@ namespace BTCPayServer.Services.Wallets
             await _Client.TrackAsync(derivationStrategy);
         }
 
-        public Task<TransactionResult> GetTransactionAsync(uint256 txId, CancellationToken cancellation = default(CancellationToken))
+        public async Task<TransactionResult> GetTransactionAsync(uint256 txId, CancellationToken cancellation = default(CancellationToken))
         {
             if (txId == null)
                 throw new ArgumentNullException(nameof(txId));
-            return _Client.GetTransactionAsync(txId, cancellation);
+            var tx = _Cache.GetTransaction(txId);
+            if (tx != null)
+                return tx;
+            tx = await _Client.GetTransactionAsync(txId, cancellation);
+            _Cache.AddToCache(tx);
+            return tx;
         }
 
         public async Task<NetworkCoins> GetCoins(DerivationStrategyBase strategy, KnownState state, CancellationToken cancellation = default(CancellationToken))

BTCPayServer/Services/Wallets/BTCPayWalletProvider.cs

@@ -10,11 +10,15 @@ namespace BTCPayServer.Services.Wallets
     {
         private ExplorerClientProvider _Client;
         BTCPayNetworkProvider _NetworkProvider;
-        public BTCPayWalletProvider(ExplorerClientProvider client, BTCPayNetworkProvider networkProvider)
+        TransactionCacheProvider _TransactionCacheProvider;
+        public BTCPayWalletProvider(ExplorerClientProvider client,
+                                    TransactionCacheProvider transactionCacheProvider,
+                                    BTCPayNetworkProvider networkProvider)
         {
             if (client == null)
                 throw new ArgumentNullException(nameof(client));
             _Client = client;
+            _TransactionCacheProvider = transactionCacheProvider;
             _NetworkProvider = networkProvider;
         }
 
@@ -32,7 +36,7 @@ namespace BTCPayServer.Services.Wallets
             var client = _Client.GetExplorerClient(cryptoCode);
             if (network == null || client == null)
                 return null;
-            return new BTCPayWallet(client, network);
+            return new BTCPayWallet(client, _TransactionCacheProvider.GetTransactionCache(network), network);
         }
     }
 }