Add ScanUTXOSet support

BTCPayServer.Tests/BTCPayServerTester.cs

@@ -198,7 +198,7 @@ namespace BTCPayServer.Tests
             return _Host.Services.GetRequiredService<T>();
         }
 
-        public T GetController<T>(string userId = null, string storeId = null) where T : Controller
+        public T GetController<T>(string userId = null, string storeId = null, Claim[] additionalClaims = null) where T : Controller
         {
             var context = new DefaultHttpContext();
             context.Request.Host = new HostString("127.0.0.1", Port);
@@ -206,7 +206,11 @@ namespace BTCPayServer.Tests
             context.Request.Protocol = "http";
             if (userId != null)
             {
-                context.User = new ClaimsPrincipal(new ClaimsIdentity(new[] { new Claim(ClaimTypes.NameIdentifier, userId) }, Policies.CookieAuthentication));
+                List<Claim> claims = new List<Claim>();
+                claims.Add(new Claim(ClaimTypes.NameIdentifier, userId));
+                if (additionalClaims != null)
+                    claims.AddRange(additionalClaims);
+                context.User = new ClaimsPrincipal(new ClaimsIdentity(claims.ToArray(), Policies.CookieAuthentication));
             }
             if (storeId != null)
             {

BTCPayServer.Tests/UnitTest1.cs

@@ -41,6 +41,9 @@ using BTCPayServer.Validation;
 using ExchangeSharp;
 using System.Security.Cryptography.X509Certificates;
 using BTCPayServer.Lightning;
+using BTCPayServer.Models.WalletViewModels;
+using System.Security.Claims;
+using BTCPayServer.Security;
 
 namespace BTCPayServer.Tests
 {
@@ -569,6 +572,72 @@ namespace BTCPayServer.Tests
             }
         }
 
+        [Fact]
+        public void CanRescanWallet()
+        {
+            using (var tester = ServerTester.Create())
+            {
+                tester.Start();
+                var acc = tester.NewAccount();
+                acc.GrantAccess();
+                acc.RegisterDerivationScheme("BTC");
+                var btcDerivationScheme = acc.DerivationScheme;
+                acc.RegisterDerivationScheme("LTC");
+                
+                var walletController = tester.PayTester.GetController<WalletsController>(acc.UserId);
+                WalletId walletId = new WalletId(acc.StoreId, "LTC");
+                var rescan = Assert.IsType<RescanWalletModel>(Assert.IsType<ViewResult>(walletController.WalletRescan(walletId).Result).Model);
+                Assert.False(rescan.Ok);
+                Assert.True(rescan.IsFullySync);
+                Assert.False(rescan.IsSupportedByCurrency);
+                Assert.False(rescan.IsServerAdmin);
+
+                walletId = new WalletId(acc.StoreId, "BTC");
+                var serverAdminClaim = new[] { new Claim(Policies.CanModifyServerSettings.Key, "true") };
+                walletController = tester.PayTester.GetController<WalletsController>(acc.UserId, additionalClaims: serverAdminClaim);
+                rescan = Assert.IsType<RescanWalletModel>(Assert.IsType<ViewResult>(walletController.WalletRescan(walletId).Result).Model);
+                Assert.True(rescan.Ok);
+                Assert.True(rescan.IsFullySync);
+                Assert.True(rescan.IsSupportedByCurrency);
+                Assert.True(rescan.IsServerAdmin);
+
+                rescan.GapLimit = 100;
+
+                // Sending a coin
+                var txId = tester.ExplorerNode.SendToAddress(btcDerivationScheme.Derive(new KeyPath("0/90")).ScriptPubKey, Money.Coins(1.0m));
+                tester.ExplorerNode.Generate(1);
+                var transactions = Assert.IsType<ListTransactionsViewModel>(Assert.IsType<ViewResult>(walletController.WalletTransactions(walletId).Result).Model);
+                Assert.Empty(transactions.Transactions);
+
+                Assert.IsType<RedirectToActionResult>(walletController.WalletRescan(walletId, rescan).Result);
+
+                while(true)
+                {
+                    rescan = Assert.IsType<RescanWalletModel>(Assert.IsType<ViewResult>(walletController.WalletRescan(walletId).Result).Model);
+                    if(rescan.Progress == null && rescan.LastSuccess != null)
+                    {
+                        if (rescan.LastSuccess.Found == 0)
+                            continue;
+                        // Scan over
+                        break;
+                    }
+                    else
+                    {
+                        Assert.Null(rescan.TimeOfScan);
+                        Assert.NotNull(rescan.RemainingTime);
+                        Assert.NotNull(rescan.Progress);
+                        Thread.Sleep(100);
+                    }
+                }
+                Assert.Null(rescan.PreviousError);
+                Assert.NotNull(rescan.TimeOfScan);
+                Assert.Equal(1, rescan.LastSuccess.Found);
+                transactions = Assert.IsType<ListTransactionsViewModel>(Assert.IsType<ViewResult>(walletController.WalletTransactions(walletId).Result).Model);
+                var tx = Assert.Single(transactions.Transactions);
+                Assert.Equal(tx.Id, txId.ToString());
+            }
+        }
+
         [Fact]
         public void CanListInvoices()
         {

BTCPayServer.Tests/docker-compose.yml

@@ -65,7 +65,7 @@ services:
 
 
   nbxplorer:
-    image: nicolasdorier/nbxplorer:1.1.0.1
+    image: nicolasdorier/nbxplorer:1.1.0.3
     ports:
       - "32838:32838"
     expose: 

BTCPayServer/BTCPayServer.csproj

@@ -47,7 +47,7 @@
     <PackageReference Include="NBitcoin" Version="4.1.1.66" />
     <PackageReference Include="NBitpayClient" Version="1.0.0.30" />	  
     <PackageReference Include="DBreeze" Version="1.87.0" />
-    <PackageReference Include="NBXplorer.Client" Version="1.0.3.2" />
+    <PackageReference Include="NBXplorer.Client" Version="1.0.3.3" />
     <PackageReference Include="NicolasDorier.CommandLine" Version="1.0.0.2" />
     <PackageReference Include="NicolasDorier.CommandLine.Configuration" Version="1.0.0.3" />
     <PackageReference Include="NicolasDorier.RateLimits" Version="1.0.0.3" />
@@ -151,6 +151,9 @@
 	  <Content Update="Views\Wallets\ListWallets.cshtml">
 	    <Pack>$(IncludeRazorContentInPack)</Pack>
 	  </Content>
+	  <Content Update="Views\Wallets\WalletRescan.cshtml">
+	    <Pack>$(IncludeRazorContentInPack)</Pack>
+	  </Content>
 	  <Content Update="Views\Wallets\WalletTransactions.cshtml">
 	    <Pack>$(IncludeRazorContentInPack)</Pack>
 	  </Content>

BTCPayServer/Controllers/WalletsController.cs

@@ -24,6 +24,7 @@ using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
 using NBitcoin;
 using NBXplorer.DerivationStrategy;
+using NBXplorer.Models;
 using Newtonsoft.Json;
 using static BTCPayServer.Controllers.StoresController;
 
@@ -184,6 +185,73 @@ namespace BTCPayServer.Controllers
             return View(model);
         }
 
+        [HttpGet]
+        [Route("{walletId}/rescan")]
+        public async Task<IActionResult> WalletRescan(
+            [ModelBinder(typeof(WalletIdModelBinder))]
+            WalletId walletId)
+        {
+            if (walletId?.StoreId == null)
+                return NotFound();
+            var store = await Repository.FindStore(walletId.StoreId, GetUserId());
+            DerivationStrategy paymentMethod = GetPaymentMethod(walletId, store);
+            if (paymentMethod == null)
+                return NotFound();
+
+            var vm = new RescanWalletModel();
+            vm.IsFullySync = _dashboard.IsFullySynched();
+            vm.IsServerAdmin = User.Claims.Any(c => c.Type == Policies.CanModifyServerSettings.Key && c.Value == "true");
+            vm.IsSupportedByCurrency = _dashboard.Get(walletId.CryptoCode)?.Status?.BitcoinStatus?.Capabilities?.CanScanTxoutSet == true;
+            var explorer = ExplorerClientProvider.GetExplorerClient(walletId.CryptoCode);
+            var scanProgress = await explorer.GetScanUTXOSetInformationAsync(paymentMethod.DerivationStrategyBase);
+            if(scanProgress != null)
+            {
+                vm.PreviousError = scanProgress.Error;
+                if (scanProgress.Status == ScanUTXOStatus.Queued || scanProgress.Status == ScanUTXOStatus.Pending)
+                {
+                    if (scanProgress.Progress == null)
+                    {
+                        vm.Progress = 0;
+                    }
+                    else
+                    {
+                        vm.Progress = scanProgress.Progress.OverallProgress;
+                        vm.RemainingTime = TimeSpan.FromSeconds(scanProgress.Progress.RemainingSeconds).PrettyPrint();
+                    }
+                }
+                if (scanProgress.Status == ScanUTXOStatus.Complete)
+                {
+                    vm.LastSuccess = scanProgress.Progress;
+                    vm.TimeOfScan = (scanProgress.Progress.CompletedAt.Value - scanProgress.Progress.StartedAt).PrettyPrint();
+                }
+            }
+            return View(vm);
+        }
+
+        [HttpPost]
+        [Route("{walletId}/rescan")]
+        public async Task<IActionResult> WalletRescan(
+            [ModelBinder(typeof(WalletIdModelBinder))]
+            WalletId walletId, RescanWalletModel vm)
+        {
+            if (walletId?.StoreId == null)
+                return NotFound();
+            var store = await Repository.FindStore(walletId.StoreId, GetUserId());
+            DerivationStrategy paymentMethod = GetPaymentMethod(walletId, store);
+            if (paymentMethod == null)
+                return NotFound();
+            var explorer = ExplorerClientProvider.GetExplorerClient(walletId.CryptoCode);
+            try
+            {
+                await explorer.ScanUTXOSetAsync(paymentMethod.DerivationStrategyBase, vm.BatchSize, vm.GapLimit, vm.StartingIndex);
+            }
+            catch (NBXplorerException ex) when (ex.Error.Code == "scanutxoset-in-progress")
+            {
+
+            }
+            return RedirectToAction();
+        }
+
         private string GetCurrencyCode(string defaultLang)
         {
             if (defaultLang == null)

BTCPayServer/HostedServices/NBXplorerWaiter.cs

@@ -47,7 +47,11 @@ namespace BTCPayServer.HostedServices
                    summary.Status != null && 
                    summary.Status.IsFullySynched;
         }
-
+        public NBXplorerSummary Get(string cryptoCode)
+        {
+            _Summaries.TryGetValue(cryptoCode, out var summary);
+            return summary;
+        }
         public IEnumerable<NBXplorerSummary> GetAll()
         {
             return _Summaries.Values;

BTCPayServer/Models/WalletViewModels/RescanWalletModel.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.Linq;
+using System.Threading.Tasks;
+using NBXplorer.Models;
+
+namespace BTCPayServer.Models.WalletViewModels
+{
+    public class RescanWalletModel
+    {
+        public bool IsServerAdmin { get; set; }
+        public bool IsSupportedByCurrency { get; set; }
+        public bool IsFullySync { get; set; }
+        public bool Ok => IsServerAdmin && IsSupportedByCurrency && IsFullySync;
+
+        [Range(1000, 10_000)]
+        public int BatchSize { get; set; } = 3000;
+        [Range(0, 10_000_000)]
+        public int StartingIndex { get; set; } = 0;
+
+        [Range(100, 100000)]
+        public int GapLimit { get; set; } = 10000;
+
+        public int? Progress { get; set; }
+        public string PreviousError { get; set; }
+        public ScanUTXOProgress LastSuccess { get; internal set; }
+        public string TimeOfScan { get; internal set; }
+        public string RemainingTime { get; internal set; }
+    }
+}

BTCPayServer/Views/Wallets/WalletRescan.cshtml

@@ -0,0 +1,104 @@
+@model RescanWalletModel
+@{
+    Layout = "../Shared/_NavLayout.cshtml";
+    ViewData["Title"] = "Rescan wallet";
+    ViewData.SetActivePageAndTitle(WalletsNavPages.Rescan);
+}
+
+<h4>@ViewData["Title"]</h4>
+
+@if (!Model.Ok)
+{
+    <div class="row">
+        <div class="col-md-10">
+            <p>This feature is disabled</p>
+            @if (Model.IsFullySync)
+            {
+                <p><span class="fa fa-check-circle" style="color:green;"></span> <span>The full node is synched</span></p>
+            }
+            else
+            {
+                <p><span class="fa fa-times-circle" style="color:red;"></span> <span>The full node is not synched</span></p>
+            }
+            @if (Model.IsServerAdmin)
+            {
+                <p><span class="fa fa-check-circle" style="color:green;"></span> <span>You are server administrator</span></p>
+            }
+            else
+            {
+                <p><span class="fa fa-times-circle" style="color:red;"></span> <span>You are not server administrator</span></p>
+            }
+            @if (Model.IsSupportedByCurrency)
+            {
+                <p><span class="fa fa-check-circle" style="color:green;"></span> <span>This full node support rescan of the UTXO set</span></p>
+            }
+            else
+            {
+                <p><span class="fa fa-times-circle" style="color:red;"></span> <span>This full node do not support rescan of the UTXO set</span></p>
+            }
+        </div>
+    </div>
+}
+else if (!Model.Progress.HasValue)
+{
+    <div class="row">
+        @if (Model.PreviousError != null)
+        {
+            <div class="alert alert-danger alert-dismissible" role="alert">
+                <button type="button" class="close" data-dismiss="alert" aria-label="Close"><span aria-hidden="true">&times;</span></button>
+                <span>The previous scan stopped with an error: @Model.PreviousError</span>
+            </div>
+        }
+        else if (Model.LastSuccess != null)
+        {
+            <div class="alert alert-success alert-dismissible" role="alert">
+                <button type="button" class="close" data-dismiss="alert" aria-label="Close"><span aria-hidden="true">&times;</span></button>
+                <span>The previous scan completed and found <b>@Model.LastSuccess.Found</b> UTXOs in <b>@Model.TimeOfScan</b> (The total UTXO set size is @Model.LastSuccess.TotalSizeOfUTXOSet.Value)</span>
+            </div>
+        }
+        <div class="col-md-10">
+            <p>
+                Scanning the UTXO set allow you to restore the balance of your wallet, but not all the transaction history.
+            </p>
+            <p>
+                This operation will scan the HD Path <b>0/*</b>, <b>1/*</b> and <b>*</b> from a starting index, until no UTXO are found in a whole gap limit.
+            </p>
+            <p>The batch size make sure the scan do not consume too much RAM at once by rescanning several time with smaller subset of addresses.</p>
+            <p>If you do not understand above, just keep the default values and click on <b>Start Scan</b></p>
+        </div>
+    </div>
+
+    <div class="row">
+        <div class="col-md-10">
+            <form method="post">
+                <div class="form-group">
+                    <label asp-for="StartingIndex"></label>
+                    <input asp-for="StartingIndex" class="form-control" type="number" />
+                </div>
+                <div class="form-group">
+                    <label asp-for="GapLimit"></label>
+                    <input asp-for="GapLimit" class="form-control" type="number" />
+                </div>
+                <div class="form-group">
+                    <label asp-for="BatchSize"></label>
+                    <input asp-for="BatchSize" class="form-control" type="number" />
+                </div>
+                <button type="submit" class="btn btn-primary">Start scan</button>
+            </form>
+        </div>
+    </div>
+}
+else
+{
+    <div class="row">
+        <div class="col-md-10">
+            <p>Scanning in progress, refresh the page to see the progress...</p>
+            <div class="progress">
+                <div class="progress-bar" role="progressbar" aria-valuenow="@(Model.Progress.Value)"
+                     aria-valuemin="0" aria-valuemax="100" style="white-space: nowrap; overflow: hidden; text-overflow: ellipsis; width:@(Model.Progress.Value)%;">
+                    @(Model.Progress.Value)% (estimated time: @Model.RemainingTime)
+                </div>
+            </div>
+        </div>
+    </div>
+}

BTCPayServer/Views/Wallets/WalletTransactions.cshtml

@@ -18,6 +18,11 @@
 </style>
 
 <h4>@ViewData["Title"]</h4>
+<div class="row">
+    <div class="col-md-10">
+        If this wallet got restored, should have received money but nothing is showing up, please <a asp-action="WalletRescan">Rescan it</a>.
+    </div>
+</div>
 <div class="row">
     <div class="col-md-10">
         <table class="table table-sm table-responsive-lg">
@@ -29,7 +34,7 @@
                 </tr>
             </thead>
             <tbody>
-                @foreach(var transaction in Model.Transactions)
+                @foreach (var transaction in Model.Transactions)
                 {
                     <tr>
                         <td>@transaction.Timestamp.ToBrowserDate()</td>
@@ -38,7 +43,7 @@
                                 @transaction.Id
                             </a>
                         </td>
-                        @if(transaction.Positive)
+                        @if (transaction.Positive)
                         {
                             <td style="text-align:right; color:green;">@transaction.Balance</td>
                         }

BTCPayServer/Views/Wallets/WalletsNavPages.cs

@@ -8,6 +8,7 @@ namespace BTCPayServer.Views.Wallets
     public enum WalletsNavPages
     {
         Send,
-        Transactions
+        Transactions,
+        Rescan
     }
 }

BTCPayServer/Views/Wallets/_Nav.cshtml

@@ -3,5 +3,6 @@
 <div class="nav flex-column nav-pills">
     <a class="nav-link @ViewData.IsActivePage(WalletsNavPages.Transactions)" asp-action="WalletTransactions">Transactions</a>
     <a class="nav-link @ViewData.IsActivePage(WalletsNavPages.Send)" asp-action="WalletSend">Send</a>
+    <a class="nav-link @ViewData.IsActivePage(WalletsNavPages.Rescan)" asp-action="WalletRescan">Rescan</a>
 </div>