From 2896a9b26fe193df4f5542151e0817997b1ff7b7 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 26 Oct 2018 23:07:39 +0900 Subject: [PATCH] Add ScanUTXOSet support --- BTCPayServer.Tests/BTCPayServerTester.cs | 8 +- BTCPayServer.Tests/UnitTest1.cs | 69 ++++++++++++ BTCPayServer.Tests/docker-compose.yml | 2 +- BTCPayServer/BTCPayServer.csproj | 5 +- BTCPayServer/Controllers/WalletsController.cs | 68 ++++++++++++ .../HostedServices/NBXplorerWaiter.cs | 6 +- .../WalletViewModels/RescanWalletModel.cs | 31 ++++++ .../Views/Wallets/WalletRescan.cshtml | 104 ++++++++++++++++++ .../Views/Wallets/WalletTransactions.cshtml | 9 +- BTCPayServer/Views/Wallets/WalletsNavPages.cs | 3 +- BTCPayServer/Views/Wallets/_Nav.cshtml | 1 + 11 files changed, 298 insertions(+), 8 deletions(-) create mode 100644 BTCPayServer/Models/WalletViewModels/RescanWalletModel.cs create mode 100644 BTCPayServer/Views/Wallets/WalletRescan.cshtml diff --git a/BTCPayServer.Tests/BTCPayServerTester.cs b/BTCPayServer.Tests/BTCPayServerTester.cs index 053e614f1..1c8d6119e 100644 --- a/BTCPayServer.Tests/BTCPayServerTester.cs +++ b/BTCPayServer.Tests/BTCPayServerTester.cs @@ -198,7 +198,7 @@ namespace BTCPayServer.Tests return _Host.Services.GetRequiredService(); } - public T GetController(string userId = null, string storeId = null) where T : Controller + public T GetController(string userId = null, string storeId = null, Claim[] additionalClaims = null) where T : Controller { var context = new DefaultHttpContext(); context.Request.Host = new HostString("127.0.0.1", Port); @@ -206,7 +206,11 @@ namespace BTCPayServer.Tests context.Request.Protocol = "http"; if (userId != null) { - context.User = new ClaimsPrincipal(new ClaimsIdentity(new[] { new Claim(ClaimTypes.NameIdentifier, userId) }, Policies.CookieAuthentication)); + List claims = new List(); + claims.Add(new Claim(ClaimTypes.NameIdentifier, userId)); + if (additionalClaims != null) + claims.AddRange(additionalClaims); + context.User = new ClaimsPrincipal(new ClaimsIdentity(claims.ToArray(), Policies.CookieAuthentication)); } if (storeId != null) { diff --git a/BTCPayServer.Tests/UnitTest1.cs b/BTCPayServer.Tests/UnitTest1.cs index 6b4b9e32a..1f2965482 100644 --- a/BTCPayServer.Tests/UnitTest1.cs +++ b/BTCPayServer.Tests/UnitTest1.cs @@ -41,6 +41,9 @@ using BTCPayServer.Validation; using ExchangeSharp; using System.Security.Cryptography.X509Certificates; using BTCPayServer.Lightning; +using BTCPayServer.Models.WalletViewModels; +using System.Security.Claims; +using BTCPayServer.Security; namespace BTCPayServer.Tests { @@ -569,6 +572,72 @@ namespace BTCPayServer.Tests } } + [Fact] + public void CanRescanWallet() + { + using (var tester = ServerTester.Create()) + { + tester.Start(); + var acc = tester.NewAccount(); + acc.GrantAccess(); + acc.RegisterDerivationScheme("BTC"); + var btcDerivationScheme = acc.DerivationScheme; + acc.RegisterDerivationScheme("LTC"); + + var walletController = tester.PayTester.GetController(acc.UserId); + WalletId walletId = new WalletId(acc.StoreId, "LTC"); + var rescan = Assert.IsType(Assert.IsType(walletController.WalletRescan(walletId).Result).Model); + Assert.False(rescan.Ok); + Assert.True(rescan.IsFullySync); + Assert.False(rescan.IsSupportedByCurrency); + Assert.False(rescan.IsServerAdmin); + + walletId = new WalletId(acc.StoreId, "BTC"); + var serverAdminClaim = new[] { new Claim(Policies.CanModifyServerSettings.Key, "true") }; + walletController = tester.PayTester.GetController(acc.UserId, additionalClaims: serverAdminClaim); + rescan = Assert.IsType(Assert.IsType(walletController.WalletRescan(walletId).Result).Model); + Assert.True(rescan.Ok); + Assert.True(rescan.IsFullySync); + Assert.True(rescan.IsSupportedByCurrency); + Assert.True(rescan.IsServerAdmin); + + rescan.GapLimit = 100; + + // Sending a coin + var txId = tester.ExplorerNode.SendToAddress(btcDerivationScheme.Derive(new KeyPath("0/90")).ScriptPubKey, Money.Coins(1.0m)); + tester.ExplorerNode.Generate(1); + var transactions = Assert.IsType(Assert.IsType(walletController.WalletTransactions(walletId).Result).Model); + Assert.Empty(transactions.Transactions); + + Assert.IsType(walletController.WalletRescan(walletId, rescan).Result); + + while(true) + { + rescan = Assert.IsType(Assert.IsType(walletController.WalletRescan(walletId).Result).Model); + if(rescan.Progress == null && rescan.LastSuccess != null) + { + if (rescan.LastSuccess.Found == 0) + continue; + // Scan over + break; + } + else + { + Assert.Null(rescan.TimeOfScan); + Assert.NotNull(rescan.RemainingTime); + Assert.NotNull(rescan.Progress); + Thread.Sleep(100); + } + } + Assert.Null(rescan.PreviousError); + Assert.NotNull(rescan.TimeOfScan); + Assert.Equal(1, rescan.LastSuccess.Found); + transactions = Assert.IsType(Assert.IsType(walletController.WalletTransactions(walletId).Result).Model); + var tx = Assert.Single(transactions.Transactions); + Assert.Equal(tx.Id, txId.ToString()); + } + } + [Fact] public void CanListInvoices() { diff --git a/BTCPayServer.Tests/docker-compose.yml b/BTCPayServer.Tests/docker-compose.yml index 7dcf10865..3e409bf92 100644 --- a/BTCPayServer.Tests/docker-compose.yml +++ b/BTCPayServer.Tests/docker-compose.yml @@ -65,7 +65,7 @@ services: nbxplorer: - image: nicolasdorier/nbxplorer:1.1.0.1 + image: nicolasdorier/nbxplorer:1.1.0.3 ports: - "32838:32838" expose: diff --git a/BTCPayServer/BTCPayServer.csproj b/BTCPayServer/BTCPayServer.csproj index 1a5ef05b1..a4c86d4e5 100644 --- a/BTCPayServer/BTCPayServer.csproj +++ b/BTCPayServer/BTCPayServer.csproj @@ -47,7 +47,7 @@ - + @@ -151,6 +151,9 @@ $(IncludeRazorContentInPack) + + $(IncludeRazorContentInPack) + $(IncludeRazorContentInPack) diff --git a/BTCPayServer/Controllers/WalletsController.cs b/BTCPayServer/Controllers/WalletsController.cs index f03e0dab1..b719fb76c 100644 --- a/BTCPayServer/Controllers/WalletsController.cs +++ b/BTCPayServer/Controllers/WalletsController.cs @@ -24,6 +24,7 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using NBitcoin; using NBXplorer.DerivationStrategy; +using NBXplorer.Models; using Newtonsoft.Json; using static BTCPayServer.Controllers.StoresController; @@ -184,6 +185,73 @@ namespace BTCPayServer.Controllers return View(model); } + [HttpGet] + [Route("{walletId}/rescan")] + public async Task WalletRescan( + [ModelBinder(typeof(WalletIdModelBinder))] + WalletId walletId) + { + if (walletId?.StoreId == null) + return NotFound(); + var store = await Repository.FindStore(walletId.StoreId, GetUserId()); + DerivationStrategy paymentMethod = GetPaymentMethod(walletId, store); + if (paymentMethod == null) + return NotFound(); + + var vm = new RescanWalletModel(); + vm.IsFullySync = _dashboard.IsFullySynched(); + vm.IsServerAdmin = User.Claims.Any(c => c.Type == Policies.CanModifyServerSettings.Key && c.Value == "true"); + vm.IsSupportedByCurrency = _dashboard.Get(walletId.CryptoCode)?.Status?.BitcoinStatus?.Capabilities?.CanScanTxoutSet == true; + var explorer = ExplorerClientProvider.GetExplorerClient(walletId.CryptoCode); + var scanProgress = await explorer.GetScanUTXOSetInformationAsync(paymentMethod.DerivationStrategyBase); + if(scanProgress != null) + { + vm.PreviousError = scanProgress.Error; + if (scanProgress.Status == ScanUTXOStatus.Queued || scanProgress.Status == ScanUTXOStatus.Pending) + { + if (scanProgress.Progress == null) + { + vm.Progress = 0; + } + else + { + vm.Progress = scanProgress.Progress.OverallProgress; + vm.RemainingTime = TimeSpan.FromSeconds(scanProgress.Progress.RemainingSeconds).PrettyPrint(); + } + } + if (scanProgress.Status == ScanUTXOStatus.Complete) + { + vm.LastSuccess = scanProgress.Progress; + vm.TimeOfScan = (scanProgress.Progress.CompletedAt.Value - scanProgress.Progress.StartedAt).PrettyPrint(); + } + } + return View(vm); + } + + [HttpPost] + [Route("{walletId}/rescan")] + public async Task WalletRescan( + [ModelBinder(typeof(WalletIdModelBinder))] + WalletId walletId, RescanWalletModel vm) + { + if (walletId?.StoreId == null) + return NotFound(); + var store = await Repository.FindStore(walletId.StoreId, GetUserId()); + DerivationStrategy paymentMethod = GetPaymentMethod(walletId, store); + if (paymentMethod == null) + return NotFound(); + var explorer = ExplorerClientProvider.GetExplorerClient(walletId.CryptoCode); + try + { + await explorer.ScanUTXOSetAsync(paymentMethod.DerivationStrategyBase, vm.BatchSize, vm.GapLimit, vm.StartingIndex); + } + catch (NBXplorerException ex) when (ex.Error.Code == "scanutxoset-in-progress") + { + + } + return RedirectToAction(); + } + private string GetCurrencyCode(string defaultLang) { if (defaultLang == null) diff --git a/BTCPayServer/HostedServices/NBXplorerWaiter.cs b/BTCPayServer/HostedServices/NBXplorerWaiter.cs index 72eb0f9dc..4a1260886 100644 --- a/BTCPayServer/HostedServices/NBXplorerWaiter.cs +++ b/BTCPayServer/HostedServices/NBXplorerWaiter.cs @@ -47,7 +47,11 @@ namespace BTCPayServer.HostedServices summary.Status != null && summary.Status.IsFullySynched; } - + public NBXplorerSummary Get(string cryptoCode) + { + _Summaries.TryGetValue(cryptoCode, out var summary); + return summary; + } public IEnumerable GetAll() { return _Summaries.Values; diff --git a/BTCPayServer/Models/WalletViewModels/RescanWalletModel.cs b/BTCPayServer/Models/WalletViewModels/RescanWalletModel.cs new file mode 100644 index 000000000..26e855279 --- /dev/null +++ b/BTCPayServer/Models/WalletViewModels/RescanWalletModel.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using System.Threading.Tasks; +using NBXplorer.Models; + +namespace BTCPayServer.Models.WalletViewModels +{ + public class RescanWalletModel + { + public bool IsServerAdmin { get; set; } + public bool IsSupportedByCurrency { get; set; } + public bool IsFullySync { get; set; } + public bool Ok => IsServerAdmin && IsSupportedByCurrency && IsFullySync; + + [Range(1000, 10_000)] + public int BatchSize { get; set; } = 3000; + [Range(0, 10_000_000)] + public int StartingIndex { get; set; } = 0; + + [Range(100, 100000)] + public int GapLimit { get; set; } = 10000; + + public int? Progress { get; set; } + public string PreviousError { get; set; } + public ScanUTXOProgress LastSuccess { get; internal set; } + public string TimeOfScan { get; internal set; } + public string RemainingTime { get; internal set; } + } +} diff --git a/BTCPayServer/Views/Wallets/WalletRescan.cshtml b/BTCPayServer/Views/Wallets/WalletRescan.cshtml new file mode 100644 index 000000000..9e2b3b4e3 --- /dev/null +++ b/BTCPayServer/Views/Wallets/WalletRescan.cshtml @@ -0,0 +1,104 @@ +@model RescanWalletModel +@{ + Layout = "../Shared/_NavLayout.cshtml"; + ViewData["Title"] = "Rescan wallet"; + ViewData.SetActivePageAndTitle(WalletsNavPages.Rescan); +} + +

@ViewData["Title"]

+ +@if (!Model.Ok) +{ +
+
+

This feature is disabled

+ @if (Model.IsFullySync) + { +

The full node is synched

+ } + else + { +

The full node is not synched

+ } + @if (Model.IsServerAdmin) + { +

You are server administrator

+ } + else + { +

You are not server administrator

+ } + @if (Model.IsSupportedByCurrency) + { +

This full node support rescan of the UTXO set

+ } + else + { +

This full node do not support rescan of the UTXO set

+ } +
+
+} +else if (!Model.Progress.HasValue) +{ +
+ @if (Model.PreviousError != null) + { + + } + else if (Model.LastSuccess != null) + { + + } +
+

+ Scanning the UTXO set allow you to restore the balance of your wallet, but not all the transaction history. +

+

+ This operation will scan the HD Path 0/*, 1/* and * from a starting index, until no UTXO are found in a whole gap limit. +

+

The batch size make sure the scan do not consume too much RAM at once by rescanning several time with smaller subset of addresses.

+

If you do not understand above, just keep the default values and click on Start Scan

+
+
+ +
+
+
+
+ + +
+
+ + +
+
+ + +
+ +
+
+
+} +else +{ +
+
+

Scanning in progress, refresh the page to see the progress...

+
+
+ @(Model.Progress.Value)% (estimated time: @Model.RemainingTime) +
+
+
+
+} diff --git a/BTCPayServer/Views/Wallets/WalletTransactions.cshtml b/BTCPayServer/Views/Wallets/WalletTransactions.cshtml index df8c3bfe6..3d1706ee1 100644 --- a/BTCPayServer/Views/Wallets/WalletTransactions.cshtml +++ b/BTCPayServer/Views/Wallets/WalletTransactions.cshtml @@ -18,6 +18,11 @@

@ViewData["Title"]

+
+
+ If this wallet got restored, should have received money but nothing is showing up, please Rescan it. +
+
@@ -29,7 +34,7 @@ - @foreach(var transaction in Model.Transactions) + @foreach (var transaction in Model.Transactions) { @@ -38,7 +43,7 @@ @transaction.Id - @if(transaction.Positive) + @if (transaction.Positive) { } diff --git a/BTCPayServer/Views/Wallets/WalletsNavPages.cs b/BTCPayServer/Views/Wallets/WalletsNavPages.cs index ed6c5bfe1..98f142778 100644 --- a/BTCPayServer/Views/Wallets/WalletsNavPages.cs +++ b/BTCPayServer/Views/Wallets/WalletsNavPages.cs @@ -8,6 +8,7 @@ namespace BTCPayServer.Views.Wallets public enum WalletsNavPages { Send, - Transactions + Transactions, + Rescan } } diff --git a/BTCPayServer/Views/Wallets/_Nav.cshtml b/BTCPayServer/Views/Wallets/_Nav.cshtml index 915dc5dad..521bf2db8 100644 --- a/BTCPayServer/Views/Wallets/_Nav.cshtml +++ b/BTCPayServer/Views/Wallets/_Nav.cshtml @@ -3,5 +3,6 @@
Transactions Send + Rescan
@transaction.Timestamp.ToBrowserDate() @transaction.Balance