From a2fa688cde8488562c376e6e8b2ec464fdeb5f89 Mon Sep 17 00:00:00 2001 From: Nicolas Dorier Date: Tue, 11 Oct 2022 17:34:29 +0900 Subject: [PATCH] Refactor labels (#4179) * Create new tables * wip * wip * Refactor LegacyLabel * Remove LabelFactory * Add migration * wip * wip * Add pull-payment attachment to tx * Address kukks points --- BTCPayServer.Client/Models/LabelData.cs | 2 + .../Models/OnChainWalletTransactionData.cs | 2 + .../Models/OnChainWalletUTXOData.cs | 2 + BTCPayServer.Data/ApplicationDbContext.cs | 8 + BTCPayServer.Data/Data/WalletData.cs | 2 + BTCPayServer.Data/Data/WalletObjectData.cs | 44 +++ .../Data/WalletObjectLinkData.cs | 61 ++++ .../Data/WalletTransactionData.cs | 2 + .../Migrations/20220929132704_label.cs | 77 +++++ .../ApplicationDbContextModelSnapshot.cs | 75 ++++- BTCPayServer.Tests/FastTests.cs | 87 ------ BTCPayServer.Tests/GreenfieldAPITests.cs | 3 +- BTCPayServer.Tests/UnitTest1.cs | 90 +++++- BTCPayServer/ColorPalette.cs | 57 ++++ ...GreenfieldStoreOnChainWalletsController.cs | 45 +-- .../Controllers/UIWalletsController.PSBT.cs | 2 +- .../Controllers/UIWalletsController.cs | 196 ++++++++---- .../BitcoinLike/BitcoinLikePayoutHandler.cs | 17 +- BTCPayServer/Data/WalletDataExtensions.cs | 27 -- .../Data/WalletTransactionDataExtensions.cs | 74 ----- BTCPayServer/Data/WalletTransactionInfo.cs | 87 ++++++ .../DbMigrationsHostedService.cs | 236 ++++++++++++++- .../TransactionLabelMarkerHostedService.cs | 129 +------- BTCPayServer/Hosting/BTCPayServerServices.cs | 1 - BTCPayServer/Hosting/MigrationStartupTask.cs | 60 ++++ .../ListTransactionsViewModel.cs | 4 +- .../WalletViewModels/TransactionTagModel.cs} | 16 +- .../WalletViewModels/WalletSendModel.cs | 2 +- .../PayJoin/PayJoinEndpointController.cs | 26 +- .../OnChainAutomatedPayoutProcessor.cs | 11 +- BTCPayServer/Services/Attachment.cs | 59 ++++ BTCPayServer/Services/Labels/Label.cs | 37 +-- BTCPayServer/Services/Labels/LabelFactory.cs | 197 ------------ BTCPayServer/Services/MigrationSettings.cs | 2 + BTCPayServer/Services/WalletRepository.cs | 280 +++++++++++++++--- .../Wallets/Export/TransactionsExport.cs | 2 +- .../UIWallets/_WalletTransactionsList.cshtml | 6 +- ...agger.template.stores-wallet.on-chain.json | 4 + 38 files changed, 1303 insertions(+), 729 deletions(-) create mode 100644 BTCPayServer.Data/Data/WalletObjectData.cs create mode 100644 BTCPayServer.Data/Data/WalletObjectLinkData.cs create mode 100644 BTCPayServer.Data/Migrations/20220929132704_label.cs create mode 100644 BTCPayServer/ColorPalette.cs delete mode 100644 BTCPayServer/Data/WalletDataExtensions.cs delete mode 100644 BTCPayServer/Data/WalletTransactionDataExtensions.cs create mode 100644 BTCPayServer/Data/WalletTransactionInfo.cs rename BTCPayServer/{Services/Labels/ColoredLabel.cs => Models/WalletViewModels/TransactionTagModel.cs} (68%) create mode 100644 BTCPayServer/Services/Attachment.cs delete mode 100644 BTCPayServer/Services/Labels/LabelFactory.cs diff --git a/BTCPayServer.Client/Models/LabelData.cs b/BTCPayServer.Client/Models/LabelData.cs index 36c87260d..b60591c8c 100644 --- a/BTCPayServer.Client/Models/LabelData.cs +++ b/BTCPayServer.Client/Models/LabelData.cs @@ -1,9 +1,11 @@ +using System; using System.Collections.Generic; using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace BTCPayServer.Client.Models { + [Obsolete] public class LabelData { public string Type { get; set; } diff --git a/BTCPayServer.Client/Models/OnChainWalletTransactionData.cs b/BTCPayServer.Client/Models/OnChainWalletTransactionData.cs index e3b21be4d..f2a179b77 100644 --- a/BTCPayServer.Client/Models/OnChainWalletTransactionData.cs +++ b/BTCPayServer.Client/Models/OnChainWalletTransactionData.cs @@ -13,7 +13,9 @@ namespace BTCPayServer.Client.Models public uint256 TransactionHash { get; set; } public string Comment { get; set; } +#pragma warning disable CS0612 // Type or member is obsolete public Dictionary Labels { get; set; } = new Dictionary(); +#pragma warning restore CS0612 // Type or member is obsolete [JsonConverter(typeof(NumericStringJsonConverter))] public decimal Amount { get; set; } diff --git a/BTCPayServer.Client/Models/OnChainWalletUTXOData.cs b/BTCPayServer.Client/Models/OnChainWalletUTXOData.cs index e50db0537..f52519d76 100644 --- a/BTCPayServer.Client/Models/OnChainWalletUTXOData.cs +++ b/BTCPayServer.Client/Models/OnChainWalletUTXOData.cs @@ -15,7 +15,9 @@ namespace BTCPayServer.Client.Models [JsonConverter(typeof(OutpointJsonConverter))] public OutPoint Outpoint { get; set; } public string Link { get; set; } +#pragma warning disable CS0612 // Type or member is obsolete public Dictionary Labels { get; set; } +#pragma warning restore CS0612 // Type or member is obsolete [JsonConverter(typeof(DateTimeToUnixTimeConverter))] public DateTimeOffset Timestamp { get; set; } [JsonConverter(typeof(KeyPathJsonConverter))] diff --git a/BTCPayServer.Data/ApplicationDbContext.cs b/BTCPayServer.Data/ApplicationDbContext.cs index 32027f397..128d9b04f 100644 --- a/BTCPayServer.Data/ApplicationDbContext.cs +++ b/BTCPayServer.Data/ApplicationDbContext.cs @@ -59,7 +59,11 @@ namespace BTCPayServer.Data public DbSet U2FDevices { get; set; } public DbSet Fido2Credentials { get; set; } public DbSet UserStore { get; set; } + [Obsolete] public DbSet Wallets { get; set; } + public DbSet WalletObjects { get; set; } + public DbSet WalletObjectLinks { get; set; } + [Obsolete] public DbSet WalletTransactions { get; set; } public DbSet WebhookDeliveries { get; set; } public DbSet Webhooks { get; set; } @@ -109,7 +113,11 @@ namespace BTCPayServer.Data Fido2Credential.OnModelCreating(builder); BTCPayServer.Data.UserStore.OnModelCreating(builder); //WalletData.OnModelCreating(builder); + WalletObjectData.OnModelCreating(builder, Database); + WalletObjectLinkData.OnModelCreating(builder, Database); +#pragma warning disable CS0612 // Type or member is obsolete WalletTransactionData.OnModelCreating(builder); +#pragma warning restore CS0612 // Type or member is obsolete WebhookDeliveryData.OnModelCreating(builder); LightningAddressData.OnModelCreating(builder); PayoutProcessorData.OnModelCreating(builder); diff --git a/BTCPayServer.Data/Data/WalletData.cs b/BTCPayServer.Data/Data/WalletData.cs index 6e011ff4b..f4922ec6c 100644 --- a/BTCPayServer.Data/Data/WalletData.cs +++ b/BTCPayServer.Data/Data/WalletData.cs @@ -1,7 +1,9 @@ +using System; using System.Collections.Generic; namespace BTCPayServer.Data { + [Obsolete] public class WalletData { [System.ComponentModel.DataAnnotations.Key] diff --git a/BTCPayServer.Data/Data/WalletObjectData.cs b/BTCPayServer.Data/Data/WalletObjectData.cs new file mode 100644 index 000000000..c2d447e4c --- /dev/null +++ b/BTCPayServer.Data/Data/WalletObjectData.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; + +namespace BTCPayServer.Data +{ + public class WalletObjectData + { + public class Types + { + public const string Label = "label"; + public const string Tx = "tx"; + } + public string WalletId { get; set; } + public string Type { get; set; } + public string Id { get; set; } + public string Data { get; set; } + + public List ChildLinks { get; set; } + public List ParentLinks { get; set; } + + internal static void OnModelCreating(ModelBuilder builder, DatabaseFacade databaseFacade) + { + builder.Entity().HasKey(o => + new + { + o.WalletId, + o.Type, + o.Id, + }); + + if (databaseFacade.IsNpgsql()) + { + builder.Entity() + .Property(o => o.Data) + .HasColumnType("JSONB"); + } + } + } +} diff --git a/BTCPayServer.Data/Data/WalletObjectLinkData.cs b/BTCPayServer.Data/Data/WalletObjectLinkData.cs new file mode 100644 index 000000000..8c9359e65 --- /dev/null +++ b/BTCPayServer.Data/Data/WalletObjectLinkData.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; + +namespace BTCPayServer.Data +{ + public class WalletObjectLinkData + { + public string WalletId { get; set; } + public string ParentType { get; set; } + public string ParentId { get; set; } + public string ChildType { get; set; } + public string ChildId { get; set; } + public string Data { get; set; } + + public WalletObjectData Parent { get; set; } + public WalletObjectData Child { get; set; } + + internal static void OnModelCreating(ModelBuilder builder, DatabaseFacade databaseFacade) + { + builder.Entity().HasKey(o => + new + { + o.WalletId, + o.ParentType, + o.ParentId, + o.ChildType, + o.ChildId, + }); + builder.Entity().HasIndex(o => new + { + o.WalletId, + o.ChildType, + o.ChildId, + }); + + builder.Entity() + .HasOne(o => o.Parent) + .WithMany(o => o.ChildLinks) + .HasForeignKey(o => new { o.WalletId, o.ParentType, o.ParentId }) + .OnDelete(DeleteBehavior.Cascade); + + builder.Entity() + .HasOne(o => o.Child) + .WithMany(o => o.ParentLinks) + .HasForeignKey(o => new { o.WalletId, o.ChildType, o.ChildId }) + .OnDelete(DeleteBehavior.Cascade); + + if (databaseFacade.IsNpgsql()) + { + builder.Entity() + .Property(o => o.Data) + .HasColumnType("JSONB"); + } + } + } +} diff --git a/BTCPayServer.Data/Data/WalletTransactionData.cs b/BTCPayServer.Data/Data/WalletTransactionData.cs index 1b0b2c8eb..19845f139 100644 --- a/BTCPayServer.Data/Data/WalletTransactionData.cs +++ b/BTCPayServer.Data/Data/WalletTransactionData.cs @@ -1,7 +1,9 @@ +using System; using Microsoft.EntityFrameworkCore; namespace BTCPayServer.Data { + [Obsolete] public class WalletTransactionData { public string WalletDataId { get; set; } diff --git a/BTCPayServer.Data/Migrations/20220929132704_label.cs b/BTCPayServer.Data/Migrations/20220929132704_label.cs new file mode 100644 index 000000000..118028c8b --- /dev/null +++ b/BTCPayServer.Data/Migrations/20220929132704_label.cs @@ -0,0 +1,77 @@ +// +using System; +using BTCPayServer.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace BTCPayServer.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20220929132704_label")] + public partial class label : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "WalletObjects", + columns: table => new + { + WalletId = table.Column(type: "TEXT", nullable: false), + Type = table.Column(type: "TEXT", nullable: false), + Id = table.Column(type: "TEXT", nullable: false), + Data = table.Column(type: migrationBuilder.IsNpgsql() ? "JSONB" : "TEXT", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_WalletObjects", x => new { x.WalletId, x.Type, x.Id }); + }); + + migrationBuilder.CreateTable( + name: "WalletObjectLinks", + columns: table => new + { + WalletId = table.Column(type: "TEXT", nullable: false), + ParentType = table.Column(type: "TEXT", nullable: false), + ParentId = table.Column(type: "TEXT", nullable: false), + ChildType = table.Column(type: "TEXT", nullable: false), + ChildId = table.Column(type: "TEXT", nullable: false), + Data = table.Column(type: migrationBuilder.IsNpgsql() ? "JSONB" : "TEXT", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_WalletObjectLinks", x => new { x.WalletId, x.ParentType, x.ParentId, x.ChildType, x.ChildId }); + table.ForeignKey( + name: "FK_WalletObjectLinks_WalletObjects_WalletId_ChildType_ChildId", + columns: x => new { x.WalletId, x.ChildType, x.ChildId }, + principalTable: "WalletObjects", + principalColumns: new[] { "WalletId", "Type", "Id" }, + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_WalletObjectLinks_WalletObjects_WalletId_ParentType_ParentId", + columns: x => new { x.WalletId, x.ParentType, x.ParentId }, + principalTable: "WalletObjects", + principalColumns: new[] { "WalletId", "Type", "Id" }, + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_WalletObjectLinks_WalletId_ChildType_ChildId", + table: "WalletObjectLinks", + columns: new[] { "WalletId", "ChildType", "ChildId" }); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "WalletObjectLinks"); + + migrationBuilder.DropTable( + name: "WalletObjects"); + } + } +} diff --git a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs index 22d7e70a6..305d08afe 100644 --- a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs @@ -16,7 +16,7 @@ namespace BTCPayServer.Migrations protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "6.0.1"); + modelBuilder.HasAnnotation("ProductVersion", "6.0.7"); modelBuilder.Entity("BTCPayServer.Data.AddressInvoiceData", b => { @@ -189,6 +189,7 @@ namespace BTCPayServer.Migrations .HasColumnType("TEXT"); b.Property("Name") + .IsRequired() .HasMaxLength(50) .HasColumnType("TEXT"); @@ -845,6 +846,52 @@ namespace BTCPayServer.Migrations b.ToTable("Wallets"); }); + modelBuilder.Entity("BTCPayServer.Data.WalletObjectData", b => + { + b.Property("WalletId") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("TEXT"); + + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("Data") + .HasColumnType("TEXT"); + + b.HasKey("WalletId", "Type", "Id"); + + b.ToTable("WalletObjects"); + }); + + modelBuilder.Entity("BTCPayServer.Data.WalletObjectLinkData", b => + { + b.Property("WalletId") + .HasColumnType("TEXT"); + + b.Property("ParentType") + .HasColumnType("TEXT"); + + b.Property("ParentId") + .HasColumnType("TEXT"); + + b.Property("ChildType") + .HasColumnType("TEXT"); + + b.Property("ChildId") + .HasColumnType("TEXT"); + + b.Property("Data") + .HasColumnType("TEXT"); + + b.HasKey("WalletId", "ParentType", "ParentId", "ChildType", "ChildId"); + + b.HasIndex("WalletId", "ChildType", "ChildId"); + + b.ToTable("WalletObjectLinks"); + }); + modelBuilder.Entity("BTCPayServer.Data.WalletTransactionData", b => { b.Property("WalletDataId") @@ -1333,6 +1380,25 @@ namespace BTCPayServer.Migrations b.Navigation("StoreData"); }); + modelBuilder.Entity("BTCPayServer.Data.WalletObjectLinkData", b => + { + b.HasOne("BTCPayServer.Data.WalletObjectData", "Child") + .WithMany("ParentLinks") + .HasForeignKey("WalletId", "ChildType", "ChildId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("BTCPayServer.Data.WalletObjectData", "Parent") + .WithMany("ChildLinks") + .HasForeignKey("WalletId", "ParentType", "ParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Child"); + + b.Navigation("Parent"); + }); + modelBuilder.Entity("BTCPayServer.Data.WalletTransactionData", b => { b.HasOne("BTCPayServer.Data.WalletData", "WalletData") @@ -1475,6 +1541,13 @@ namespace BTCPayServer.Migrations b.Navigation("WalletTransactions"); }); + modelBuilder.Entity("BTCPayServer.Data.WalletObjectData", b => + { + b.Navigation("ChildLinks"); + + b.Navigation("ParentLinks"); + }); + modelBuilder.Entity("BTCPayServer.Data.WebhookData", b => { b.Navigation("Deliveries"); diff --git a/BTCPayServer.Tests/FastTests.cs b/BTCPayServer.Tests/FastTests.cs index c19eaea6a..91dc08538 100644 --- a/BTCPayServer.Tests/FastTests.cs +++ b/BTCPayServer.Tests/FastTests.cs @@ -483,93 +483,6 @@ namespace BTCPayServer.Tests } #endif - [Fact] - public void CanParseLegacyLabels() - { - static void AssertContainsRawLabel(WalletTransactionInfo info) - { - foreach (var item in new[] { "blah", "lol", "hello" }) - { - Assert.True(info.Labels.ContainsKey(item)); - var rawLabel = Assert.IsType(info.Labels[item]); - Assert.Equal("raw", rawLabel.Type); - Assert.Equal(item, rawLabel.Text); - } - } - var data = new WalletTransactionData(); - data.Labels = "blah,lol,hello,lol"; - var info = data.GetBlobInfo(); - Assert.Equal(3, info.Labels.Count); - AssertContainsRawLabel(info); - data.SetBlobInfo(info); - Assert.Contains("raw", data.Labels); - Assert.Contains("{", data.Labels); - Assert.Contains("[", data.Labels); - info = data.GetBlobInfo(); - AssertContainsRawLabel(info); - - - data = new WalletTransactionData() - { - Labels = "pos", - Blob = Encoders.Hex.DecodeData("1f8b08000000000000037abf7b7fb592737e6e6e6a5e89929592522d000000ffff030036bc6ad911000000") - }; - info = data.GetBlobInfo(); - var label = Assert.Single(info.Labels); - Assert.Equal("raw", label.Value.Type); - Assert.Equal("pos", label.Value.Text); - Assert.Equal("pos", label.Key); - - - static void AssertContainsLabel(WalletTransactionInfo info) - { - Assert.Equal(2, info.Labels.Count); - var invoiceLabel = Assert.IsType(info.Labels["invoice"]); - Assert.Equal("BFm1MCJPBCDeRoWXvPcwnM", invoiceLabel.Reference); - Assert.Equal("invoice", invoiceLabel.Text); - Assert.Equal("invoice", invoiceLabel.Type); - - var appLabel = Assert.IsType(info.Labels["app"]); - Assert.Equal("87kj5yKay8mB4UUZcJhZH5TqDKMD3CznjwLjiu1oYZXe", appLabel.Reference); - Assert.Equal("app", appLabel.Text); - Assert.Equal("app", appLabel.Type); - } - data = new WalletTransactionData() - { - Labels = "[\"{\\n \\\"value\\\": \\\"invoice\\\",\\n \\\"id\\\": \\\"BFm1MCJPBCDeRoWXvPcwnM\\\"\\n}\",\"{\\n \\\"value\\\": \\\"app\\\",\\n \\\"id\\\": \\\"87kj5yKay8mB4UUZcJhZH5TqDKMD3CznjwLjiu1oYZXe\\\"\\n}\"]", - }; - info = data.GetBlobInfo(); - AssertContainsLabel(info); - data.SetBlobInfo(info); - info = data.GetBlobInfo(); - AssertContainsLabel(info); - - static void AssertPayoutLabel(WalletTransactionInfo info) - { - Assert.Single(info.Labels); - var l = Assert.IsType(info.Labels["payout"]); - Assert.Single(Assert.Single(l.PullPaymentPayouts, k => k.Key == "pullPaymentId").Value, "payoutId"); - Assert.Equal("walletId", l.WalletId); - } - - var payoutId = "payoutId"; - var pullPaymentId = "pullPaymentId"; - var walletId = "walletId"; - // How it was serialized before - - data = new WalletTransactionData() - { - Labels = new JArray(JObject.FromObject(new { value = "payout", id = payoutId, pullPaymentId, walletId })).ToString() - }; - info = data.GetBlobInfo(); - AssertPayoutLabel(info); - data.SetBlobInfo(info); - info = data.GetBlobInfo(); - AssertPayoutLabel(info); - } - - - [Fact] public void DeterministicUTXOSorter() { diff --git a/BTCPayServer.Tests/GreenfieldAPITests.cs b/BTCPayServer.Tests/GreenfieldAPITests.cs index 25de776e0..afc7edb60 100644 --- a/BTCPayServer.Tests/GreenfieldAPITests.cs +++ b/BTCPayServer.Tests/GreenfieldAPITests.cs @@ -2270,6 +2270,7 @@ namespace BTCPayServer.Tests Assert.Equal(transaction.TransactionHash, txdata.TransactionHash); Assert.Equal(String.Empty, transaction.Comment); +#pragma warning disable CS0612 // Type or member is obsolete Assert.Equal(new Dictionary(), transaction.Labels); // transaction patch tests @@ -2290,7 +2291,7 @@ namespace BTCPayServer.Tests }.ToJson(), patchedTransaction.Labels.ToJson() ); - +#pragma warning restore CS0612 // Type or member is obsolete await AssertHttpError(403, async () => { await viewOnlyClient.ShowOnChainWalletTransactions(walletId.StoreId, walletId.CryptoCode); diff --git a/BTCPayServer.Tests/UnitTest1.cs b/BTCPayServer.Tests/UnitTest1.cs index 1204bcc37..82ad6fc9a 100644 --- a/BTCPayServer.Tests/UnitTest1.cs +++ b/BTCPayServer.Tests/UnitTest1.cs @@ -40,6 +40,7 @@ using BTCPayServer.Security.Bitpay; using BTCPayServer.Services; using BTCPayServer.Services.Apps; using BTCPayServer.Services.Invoices; +using BTCPayServer.Services.Labels; using BTCPayServer.Services.Mails; using BTCPayServer.Services.Rates; using BTCPayServer.Storage.Models; @@ -51,6 +52,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using NBitcoin; using NBitcoin.DataEncoders; using NBitcoin.Payment; @@ -785,9 +787,9 @@ namespace BTCPayServer.Tests tx = Assert.Single(transactions.Transactions); Assert.Equal("hello", tx.Comment); - Assert.Contains("test", tx.Labels.Select(l => l.Text)); - Assert.Contains("test2", tx.Labels.Select(l => l.Text)); - Assert.Equal(2, tx.Labels.GroupBy(l => l.Color).Count()); + Assert.Contains("test", tx.Tags.Select(l => l.Text)); + Assert.Contains("test2", tx.Tags.Select(l => l.Text)); + Assert.Equal(2, tx.Tags.GroupBy(l => l.Color).Count()); Assert.IsType( await walletController.ModifyTransaction(walletId, tx.Id, removelabel: "test2")); @@ -797,12 +799,9 @@ namespace BTCPayServer.Tests tx = Assert.Single(transactions.Transactions); Assert.Equal("hello", tx.Comment); - Assert.Contains("test", tx.Labels.Select(l => l.Text)); - Assert.DoesNotContain("test2", tx.Labels.Select(l => l.Text)); - Assert.Single(tx.Labels.GroupBy(l => l.Color)); - - var walletInfo = await tester.PayTester.GetService().GetWalletInfo(walletId); - Assert.Single(walletInfo.LabelColors); // the test2 color should have been removed + Assert.Contains("test", tx.Tags.Select(l => l.Text)); + Assert.DoesNotContain("test2", tx.Tags.Select(l => l.Text)); + Assert.Single(tx.Tags.GroupBy(l => l.Color)); } [Fact(Timeout = LongRunningTestTimeout)] @@ -2521,6 +2520,79 @@ namespace BTCPayServer.Tests Assert.True(lnMethod.IsInternalNode); } + [Fact(Timeout = LongRunningTestTimeout)] + [Trait("Integration", "Integration")] + [Obsolete] + public async Task CanDoLabelMigrations() + { + using var tester = CreateServerTester(newDb: true); + await tester.StartAsync(); + var dbf = tester.PayTester.GetService(); + int walletCount = 1000; + var wallet = "walletttttttttttttttttttttttttttt"; + using (var db = dbf.CreateContext()) + { + for (int i = 0; i < walletCount; i++) + { + var walletData = new WalletData() { Id = $"S-{wallet}{i}-BTC" }; + walletData.Blob = ZipUtils.Zip("{\"LabelColors\": { \"label1\" : \"black\", \"payout\":\"green\" }}"); + db.Wallets.Add(walletData); + } + await db.SaveChangesAsync(); + } + uint256 firstTxId = null; + using (var db = dbf.CreateContext()) + { + int transactionCount = 10_000; + for (int i = 0; i < transactionCount; i++) + { + var txId = RandomUtils.GetUInt256(); + var wt = new WalletTransactionData() + { + WalletDataId = $"S-{wallet}{i % walletCount}-BTC", + TransactionId = txId.ToString(), + }; + firstTxId ??= txId; + if (i != 10) + wt.Blob = ZipUtils.Zip("{\"Comment\":\"test\"}"); + if (i % 1240 != 0) + { + wt.Labels = "[{\"type\":\"raw\", \"text\":\"label1\"}]"; + } + else if (i == 0) + { + wt.Labels = "[{\"type\":\"raw\", \"text\":\"label1\"},{\"type\":\"raw\", \"text\":\"labelo" + i + "\"}, " + + "{\"type\":\"payout\", \"text\":\"payout\", \"pullPaymentPayouts\":{\"pp1\":[\"p1\",\"p2\"],\"pp2\":[\"p3\"]}}]"; + } + else + { + wt.Labels = "[{\"type\":\"raw\", \"text\":\"label1\"},{\"type\":\"raw\", \"text\":\"labelo" + i + "\"}]"; + } + db.WalletTransactions.Add(wt); + } + await db.SaveChangesAsync(); + } + await RestartMigration(tester); + var migrator = tester.PayTester.GetService>().OfType().First(); + await migrator.MigratedTransactionLabels(0); + + var walletRepo = tester.PayTester.GetService(); + var wi1 = await walletRepo.GetWalletLabels(new WalletId($"{wallet}0", "BTC")); + Assert.Equal(3, wi1.Length); + Assert.Contains(wi1, o => o.Label == "label1" && o.Color == "black"); + Assert.Contains(wi1, o => o.Label == "labelo0" && o.Color == "#000"); + Assert.Contains(wi1, o => o.Label == "payout" && o.Color == "green"); + + var txInfo = await walletRepo.GetWalletTransactionsInfo(new WalletId($"{wallet}0", "BTC"), new[] { firstTxId.ToString() }); + Assert.Equal("test", txInfo.Values.First().Comment); + // Should have the 2 raw labels, and one legacy label for payouts + Assert.Equal(3, txInfo.Values.First().LegacyLabels.Count); + var payoutLabel = txInfo.Values.First().LegacyLabels.Select(l => l.Value).OfType().First(); + Assert.Equal(2, payoutLabel.PullPaymentPayouts.Count); + Assert.Equal(2, payoutLabel.PullPaymentPayouts["pp1"].Count); + Assert.Single(payoutLabel.PullPaymentPayouts["pp2"]); + } + [Fact(Timeout = LongRunningTestTimeout)] [Trait("Integration", "Integration")] diff --git a/BTCPayServer/ColorPalette.cs b/BTCPayServer/ColorPalette.cs new file mode 100644 index 000000000..b8c7e4efb --- /dev/null +++ b/BTCPayServer/ColorPalette.cs @@ -0,0 +1,57 @@ +using System; +using System.Drawing; +using System.Text; +using NBitcoin.Crypto; + +namespace BTCPayServer +{ + public class ColorPalette + { + public string TextColor(string bgColor) + { + int nThreshold = 105; + var bg = ColorTranslator.FromHtml(bgColor); + int bgDelta = Convert.ToInt32((bg.R * 0.299) + (bg.G * 0.587) + (bg.B * 0.114)); + Color color = (255 - bgDelta < nThreshold) ? Color.Black : Color.White; + return ColorTranslator.ToHtml(color); + } + // Borrowed from https://github.com/ManageIQ/guides/blob/master/labels.md + public static readonly ColorPalette Default = new ColorPalette(new string[] { + "#fbca04", + "#0e8a16", + "#ff7619", + "#84b6eb", + "#5319e7", + "#cdcdcd", + "#cc317c", + }); + private ColorPalette(string[] labels) + { + Labels = labels; + } + + public readonly string[] Labels; + + public string DeterministicColor(string label) + { + switch (label) + { + case "payjoin": + return "#51b13e"; + case "invoice": + return "#cedc21"; + case "payment-request": + return "#489D77"; + case "app": + return "#5093B6"; + case "pj-exposed": + return "#51b13e"; + case "payout": + return "#3F88AF"; + default: + var num = NBitcoin.Utils.ToUInt32(Hashes.SHA256(Encoding.UTF8.GetBytes(label)), 0, true); + return Labels[num % Labels.Length]; + } + } + } +} diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldStoreOnChainWalletsController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldStoreOnChainWalletsController.cs index e769134a4..c04e37358 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldStoreOnChainWalletsController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldStoreOnChainWalletsController.cs @@ -52,7 +52,6 @@ namespace BTCPayServer.Controllers.Greenfield private readonly EventAggregator _eventAggregator; private readonly WalletReceiveService _walletReceiveService; private readonly IFeeProviderFactory _feeProviderFactory; - private readonly LabelFactory _labelFactory; private readonly UTXOLocker _utxoLocker; public GreenfieldStoreOnChainWalletsController( @@ -69,7 +68,6 @@ namespace BTCPayServer.Controllers.Greenfield EventAggregator eventAggregator, WalletReceiveService walletReceiveService, IFeeProviderFactory feeProviderFactory, - LabelFactory labelFactory, UTXOLocker utxoLocker ) { @@ -86,7 +84,6 @@ namespace BTCPayServer.Controllers.Greenfield _eventAggregator = eventAggregator; _walletReceiveService = walletReceiveService; _feeProviderFactory = feeProviderFactory; - _labelFactory = labelFactory; _utxoLocker = utxoLocker; } @@ -202,7 +199,7 @@ namespace BTCPayServer.Controllers.Greenfield if (!string.IsNullOrWhiteSpace(labelFilter)) { walletTransactionsInfoAsync.TryGetValue(t.TransactionId.ToString(), out var transactionInfo); - if (transactionInfo?.Labels.ContainsKey(labelFilter) is true) + if (transactionInfo?.LabelColors.ContainsKey(labelFilter) is true) filteredList.Add(t); } if (statusFilter?.Any() is true) @@ -270,36 +267,18 @@ namespace BTCPayServer.Controllers.Greenfield } var walletId = new WalletId(storeId, cryptoCode); - var walletTransactionsInfoAsync = _walletRepository.GetWalletTransactionsInfo(walletId); - if (!(await walletTransactionsInfoAsync).TryGetValue(transactionId, out var walletTransactionInfo)) - { - walletTransactionInfo = new WalletTransactionInfo(); - } + var txObjectId = new WalletObjectId(walletId, WalletObjectData.Types.Tx, transactionId); if (request.Comment != null) { - walletTransactionInfo.Comment = request.Comment.Trim().Truncate(WalletTransactionDataExtensions.MaxCommentSize); + await _walletRepository.SetWalletObjectComment(txObjectId, request.Comment); } if (request.Labels != null) { - var walletBlobInfo = await _walletRepository.GetWalletInfo(walletId); - - foreach (string label in request.Labels) - { - var rawLabel = await _labelFactory.BuildLabel( - walletBlobInfo, - Request, - walletTransactionInfo, - walletId, - transactionId, - label - ); - walletTransactionInfo.Labels.TryAdd(rawLabel.Text, rawLabel); - } + await _walletRepository.AddWalletObjectLabels(txObjectId, request.Labels.ToArray()); } - await _walletRepository.SetWalletTransactionInfo(walletId, transactionId, walletTransactionInfo); var walletTransactionsInfo = (await _walletRepository.GetWalletTransactionsInfo(walletId, new[] { transactionId })) .Values @@ -319,19 +298,20 @@ namespace BTCPayServer.Controllers.Greenfield var wallet = _btcPayWalletProvider.GetWallet(network); var walletId = new WalletId(storeId, cryptoCode); - var walletTransactionsInfoAsync = await _walletRepository.GetWalletTransactionsInfo(walletId); var utxos = await wallet.GetUnspentCoins(derivationScheme.AccountDerivation); - + var walletTransactionsInfoAsync = await _walletRepository.GetWalletTransactionsInfo(walletId, utxos.Select(u => u.OutPoint.Hash.ToString()).ToHashSet().ToArray()); return Ok(utxos.Select(coin => { walletTransactionsInfoAsync.TryGetValue(coin.OutPoint.Hash.ToString(), out var info); - var labels = info?.Labels ?? new Dictionary(); + return new OnChainWalletUTXOData() { Outpoint = coin.OutPoint, Amount = coin.Value.GetValue(network), Comment = info?.Comment, - Labels = info?.Labels, +#pragma warning disable CS0612 // Type or member is obsolete + Labels = info?.LegacyLabels ?? new Dictionary(), +#pragma warning restore CS0612 // Type or member is obsolete Link = string.Format(CultureInfo.InvariantCulture, network.BlockExplorerLink, coin.OutPoint.Hash.ToString()), Timestamp = coin.Timestamp, @@ -592,8 +572,7 @@ namespace BTCPayServer.Controllers.Greenfield payjoinPSBT.Finalize(); var payjoinTransaction = payjoinPSBT.ExtractTransaction(); var hash = payjoinTransaction.GetHash(); - _eventAggregator.Publish(new UpdateTransactionLabel(new WalletId(Store.Id, cryptoCode), hash, - UpdateTransactionLabel.PayjoinLabelTemplate())); + await this._walletRepository.AddWalletTransactionAttachment(new WalletId(Store.Id, cryptoCode), hash, Attachment.Payjoin()); broadcastResult = await explorerClient.BroadcastAsync(payjoinTransaction); if (broadcastResult.Success) { @@ -676,7 +655,9 @@ namespace BTCPayServer.Controllers.Greenfield { TransactionHash = tx.TransactionId, Comment = walletTransactionsInfoAsync?.Comment ?? string.Empty, - Labels = walletTransactionsInfoAsync?.Labels ?? new Dictionary(), +#pragma warning disable CS0612 // Type or member is obsolete + Labels = walletTransactionsInfoAsync?.LegacyLabels ?? new Dictionary(), +#pragma warning restore CS0612 // Type or member is obsolete Amount = tx.BalanceChange.GetValue(wallet.Network), BlockHash = tx.BlockHash, BlockHeight = tx.Height, diff --git a/BTCPayServer/Controllers/UIWalletsController.PSBT.cs b/BTCPayServer/Controllers/UIWalletsController.PSBT.cs index 78216fe1a..db604b22a 100644 --- a/BTCPayServer/Controllers/UIWalletsController.PSBT.cs +++ b/BTCPayServer/Controllers/UIWalletsController.PSBT.cs @@ -460,7 +460,7 @@ namespace BTCPayServer.Controllers vm.SigningContext.OriginalPSBT = psbt.ToBase64(); proposedPayjoin.Finalize(); var hash = proposedPayjoin.ExtractTransaction().GetHash(); - _EventAggregator.Publish(new UpdateTransactionLabel(walletId, hash, UpdateTransactionLabel.PayjoinLabelTemplate())); + await WalletRepository.AddWalletTransactionAttachment(walletId, hash, Attachment.Payjoin()); TempData.SetStatusMessageModel(new StatusMessageModel { Severity = StatusMessageModel.StatusSeverity.Success, diff --git a/BTCPayServer/Controllers/UIWalletsController.cs b/BTCPayServer/Controllers/UIWalletsController.cs index cde13d3dd..38b8d51a3 100644 --- a/BTCPayServer/Controllers/UIWalletsController.cs +++ b/BTCPayServer/Controllers/UIWalletsController.cs @@ -39,6 +39,8 @@ using NBXplorer.DerivationStrategy; using NBXplorer.Models; using Newtonsoft.Json; using StoreData = BTCPayServer.Data.StoreData; +using Microsoft.AspNetCore.Routing; +using Newtonsoft.Json.Linq; namespace BTCPayServer.Controllers { @@ -60,11 +62,10 @@ namespace BTCPayServer.Controllers private readonly IFeeProviderFactory _feeRateProvider; private readonly BTCPayWalletProvider _walletProvider; private readonly WalletReceiveService _walletReceiveService; - private readonly EventAggregator _EventAggregator; private readonly SettingsRepository _settingsRepository; private readonly DelayedTransactionBroadcaster _broadcaster; private readonly PayjoinClient _payjoinClient; - private readonly LabelFactory _labelFactory; + private readonly LinkGenerator _linkGenerator; private readonly PullPaymentHostedService _pullPaymentHostedService; private readonly UTXOLocker _utxoLocker; private readonly WalletHistogramService _walletHistogramService; @@ -84,16 +85,16 @@ namespace BTCPayServer.Controllers IFeeProviderFactory feeRateProvider, BTCPayWalletProvider walletProvider, WalletReceiveService walletReceiveService, - EventAggregator eventAggregator, SettingsRepository settingsRepository, DelayedTransactionBroadcaster broadcaster, PayjoinClient payjoinClient, - LabelFactory labelFactory, IServiceProvider serviceProvider, PullPaymentHostedService pullPaymentHostedService, - UTXOLocker utxoLocker) + UTXOLocker utxoLocker, + LinkGenerator linkGenerator) { _currencyTable = currencyTable; + _linkGenerator = linkGenerator; Repository = repo; WalletRepository = walletRepository; RateFetcher = rateProvider; @@ -105,11 +106,9 @@ namespace BTCPayServer.Controllers _feeRateProvider = feeRateProvider; _walletProvider = walletProvider; _walletReceiveService = walletReceiveService; - _EventAggregator = eventAggregator; _settingsRepository = settingsRepository; _broadcaster = broadcaster; _payjoinClient = payjoinClient; - _labelFactory = labelFactory; _pullPaymentHostedService = pullPaymentHostedService; _utxoLocker = utxoLocker; ServiceProvider = serviceProvider; @@ -146,58 +145,19 @@ namespace BTCPayServer.Controllers if (paymentMethod == null) return NotFound(); - var walletBlobInfoAsync = WalletRepository.GetWalletInfo(walletId); - var walletTransactionsInfoAsync = WalletRepository.GetWalletTransactionsInfo(walletId); + var txObjId = new WalletObjectId(walletId, WalletObjectData.Types.Tx, transactionId); var wallet = _walletProvider.GetWallet(paymentMethod.Network); - var walletBlobInfo = await walletBlobInfoAsync; - var walletTransactionsInfo = await walletTransactionsInfoAsync; if (addlabel != null) { - if (!walletTransactionsInfo.TryGetValue(transactionId, out var walletTransactionInfo)) - { - walletTransactionInfo = new WalletTransactionInfo(); - } - - var rawLabel = await _labelFactory.BuildLabel( - walletBlobInfo, - Request!, - walletTransactionInfo, - walletId, - transactionId, - addlabel - ); - if (walletTransactionInfo.Labels.TryAdd(rawLabel.Text, rawLabel)) - { - await WalletRepository.SetWalletTransactionInfo(walletId, transactionId, walletTransactionInfo); - } + await WalletRepository.AddWalletObjectLabels(txObjId, addlabel); } else if (removelabel != null) { - removelabel = removelabel.Trim(); - if (walletTransactionsInfo.TryGetValue(transactionId, out var walletTransactionInfo)) - { - if (walletTransactionInfo.Labels.Remove(removelabel)) - { - var canDeleteColor = - !walletTransactionsInfo.Any(txi => txi.Value.Labels.ContainsKey(removelabel)); - if (canDeleteColor) - { - walletBlobInfo.LabelColors.Remove(removelabel); - await WalletRepository.SetWalletInfo(walletId, walletBlobInfo); - } - await WalletRepository.SetWalletTransactionInfo(walletId, transactionId, walletTransactionInfo); - } - } + await WalletRepository.RemoveWalletObjectLabels(txObjId, removelabel); } else if (addcomment != null) { - addcomment = addcomment.Trim().Truncate(WalletTransactionDataExtensions.MaxCommentSize); - if (!walletTransactionsInfo.TryGetValue(transactionId, out var walletTransactionInfo)) - { - walletTransactionInfo = new WalletTransactionInfo(); - } - walletTransactionInfo.Comment = addcomment; - await WalletRepository.SetWalletTransactionInfo(walletId, transactionId, walletTransactionInfo); + await WalletRepository.SetWalletObjectComment(txObjId, addcomment); } return RedirectToAction(nameof(WalletTransactions), new { walletId = walletId.ToString() }); } @@ -267,15 +227,17 @@ namespace BTCPayServer.Controllers return NotFound(); var wallet = _walletProvider.GetWallet(paymentMethod.Network); - var walletBlobAsync = WalletRepository.GetWalletInfo(walletId); - var walletTransactionsInfoAsync = WalletRepository.GetWalletTransactionsInfo(walletId); // We can't filter at the database level if we need to apply label filter var preFiltering = string.IsNullOrEmpty(labelFilter); var transactions = await wallet.FetchTransactionHistory(paymentMethod.AccountDerivation, preFiltering ? skip : null, preFiltering ? count : null); - var walletBlob = await walletBlobAsync; - var walletTransactionsInfo = await walletTransactionsInfoAsync; + var walletTransactionsInfo = await WalletRepository.GetWalletTransactionsInfo(walletId, transactions.Select(t => t.TransactionId.ToString()).ToArray()); var model = new ListTransactionsViewModel { Skip = skip, Count = count }; + model.Labels.AddRange( + (await WalletRepository.GetWalletLabels(walletId)) + .Select(c => (c.Label, c.Color, ColorPalette.Default.TextColor(c.Color))) + ); + if (labelFilter != null) { model.PaginationQuery = new Dictionary { { "labelFilter", labelFilter } }; @@ -305,14 +267,13 @@ namespace BTCPayServer.Controllers if (walletTransactionsInfo.TryGetValue(tx.TransactionId.ToString(), out var transactionInfo)) { - var labels = _labelFactory.ColorizeTransactionLabels(walletBlob, transactionInfo, Request); - vm.Labels.AddRange(labels); - model.Labels.AddRange(labels); + var labels = CreateTransactionTagModels(transactionInfo); + vm.Tags.AddRange(labels); vm.Comment = transactionInfo.Comment; } if (labelFilter == null || - vm.Labels.Any(l => l.Text.Equals(labelFilter, StringComparison.OrdinalIgnoreCase))) + vm.Tags.Any(l => l.Text.Equals(labelFilter, StringComparison.OrdinalIgnoreCase))) model.Transactions.Add(vm); } @@ -613,17 +574,15 @@ namespace BTCPayServer.Controllers var schemeSettings = GetDerivationSchemeSettings(walletId); if (schemeSettings is null) return NotFound(); - var walletBlobAsync = await WalletRepository.GetWalletInfo(walletId); - var walletTransactionsInfoAsync = await WalletRepository.GetWalletTransactionsInfo(walletId); var utxos = await _walletProvider.GetWallet(network) .GetUnspentCoins(schemeSettings.AccountDerivation, false, cancellation); + + var walletTransactionsInfoAsync = await this.WalletRepository.GetWalletTransactionsInfo(walletId, utxos.Select(u => u.OutPoint.Hash.ToString()).Distinct().ToArray()); vm.InputsAvailable = utxos.Select(coin => { walletTransactionsInfoAsync.TryGetValue(coin.OutPoint.Hash.ToString(), out var info); - var labels = info?.Labels == null - ? new List() - : _labelFactory.ColorizeTransactionLabels(walletBlobAsync, info, Request).ToList(); + var labels = CreateTransactionTagModels(info).ToList(); return new WalletSendModel.InputSelectionOption() { Outpoint = coin.OutPoint.ToString(), @@ -1359,6 +1318,117 @@ namespace BTCPayServer.Controllers private string GetUserId() => _userManager.GetUserId(User); private StoreData GetCurrentStore() => HttpContext.GetStoreData(); + + public IEnumerable CreateTransactionTagModels(WalletTransactionInfo? transactionInfo) + { + if (transactionInfo is null) + return Array.Empty(); + + string PayoutTooltip(IGrouping? payoutsByPullPaymentId = null) + { + if (payoutsByPullPaymentId is null) + { + return "Paid a payout"; + } + else if (payoutsByPullPaymentId.Count() == 1) + { + var pp = payoutsByPullPaymentId.Key; + var payout = payoutsByPullPaymentId.First(); + if (!string.IsNullOrEmpty(pp)) + return $"Paid a payout ({payout}) of a pull payment ({pp})"; + else + return $"Paid a payout {payout}"; + } + else + { + var pp = payoutsByPullPaymentId.Key; + if (!string.IsNullOrEmpty(pp)) + return $"Paid {payoutsByPullPaymentId.Count()} payouts of a pull payment ({pp})"; + else + return $"Paid {payoutsByPullPaymentId.Count()} payouts"; + } + } + + var models = new Dictionary(); + foreach (var tag in transactionInfo.Attachments) + { + if (models.ContainsKey(tag.Type)) + continue; + if (!transactionInfo.LabelColors.TryGetValue(tag.Type, out var color)) + continue; + var model = new TransactionTagModel + { + Text = tag.Type, + Color = color, + TextColor = ColorPalette.Default.TextColor(color) + }; + models.Add(tag.Type, model); + if (tag.Type == "payout") + { + var payoutsByPullPaymentId = + transactionInfo.Attachments.Where(t => t.Type == "payout") + .GroupBy(t => t.Data?["pullPaymentId"]?.Value() ?? "", + k => k.Id).ToList(); + + model.Tooltip = payoutsByPullPaymentId.Count switch + { + 0 => PayoutTooltip(), + 1 => PayoutTooltip(payoutsByPullPaymentId.First()), + _ => + $"
    {string.Join(string.Empty, payoutsByPullPaymentId.Select(pair => $"
  • {PayoutTooltip(pair)}
    • "))}
" + }; + + model.Link = _linkGenerator.PayoutLink(transactionInfo.WalletId.ToString(), null, PayoutState.Completed, Request.Scheme, Request.Host, + Request.PathBase); + } + else if (tag.Type == "payjoin") + { + model.Tooltip = $"This UTXO was part of a PayJoin transaction."; + } + else if (tag.Type == "invoice") + { + model.Tooltip = $"Received through an invoice {tag.Id}"; + model.Link = string.IsNullOrEmpty(tag.Id) + ? null + : _linkGenerator.InvoiceLink(tag.Id, Request.Scheme, Request.Host, Request.PathBase); + } + else if (tag.Type == "payment-request") + { + model.Tooltip = $"Received through a payment request {tag.Id}"; + model.Link = _linkGenerator.PaymentRequestLink(tag.Id, Request.Scheme, Request.Host, Request.PathBase); + } + else if (tag.Type == "app") + { + model.Tooltip = $"Received through an app {tag.Id}"; + model.Link = _linkGenerator.AppLink(tag.Id, Request.Scheme, Request.Host, Request.PathBase); + } + else if (tag.Type == "pj-exposed") + { + + if (tag.Id.Length != 0) + { + model.Tooltip = $"This UTXO was exposed through a PayJoin proposal for an invoice ({tag.Id})"; + model.Link = _linkGenerator.InvoiceLink(tag.Id, Request.Scheme, Request.Host, Request.PathBase); + } + else + { + model.Tooltip = $"This UTXO was exposed through a PayJoin proposal"; + } + } + else if (tag.Type == "payjoin") + { + model.Tooltip = $"This UTXO was part of a PayJoin transaction."; + } + } + foreach (var label in transactionInfo.LabelColors) + models.TryAdd(label.Key, new TransactionTagModel + { + Text = label.Key, + Color = label.Value, + TextColor = ColorPalette.Default.TextColor(label.Value) + }); + return models.Values.OrderBy(v => v.Text); + } } public class WalletReceiveViewModel diff --git a/BTCPayServer/Data/Payouts/BitcoinLike/BitcoinLikePayoutHandler.cs b/BTCPayServer/Data/Payouts/BitcoinLike/BitcoinLikePayoutHandler.cs index 268a228bd..4fd643839 100644 --- a/BTCPayServer/Data/Payouts/BitcoinLike/BitcoinLikePayoutHandler.cs +++ b/BTCPayServer/Data/Payouts/BitcoinLike/BitcoinLikePayoutHandler.cs @@ -34,22 +34,24 @@ public class BitcoinLikePayoutHandler : IPayoutHandler private readonly ExplorerClientProvider _explorerClientProvider; private readonly BTCPayNetworkJsonSerializerSettings _jsonSerializerSettings; private readonly ApplicationDbContextFactory _dbContextFactory; - private readonly EventAggregator _eventAggregator; private readonly NotificationSender _notificationSender; private readonly Logs Logs; + + public WalletRepository WalletRepository { get; } + public BitcoinLikePayoutHandler(BTCPayNetworkProvider btcPayNetworkProvider, + WalletRepository walletRepository, ExplorerClientProvider explorerClientProvider, BTCPayNetworkJsonSerializerSettings jsonSerializerSettings, ApplicationDbContextFactory dbContextFactory, - EventAggregator eventAggregator, NotificationSender notificationSender, Logs logs) { _btcPayNetworkProvider = btcPayNetworkProvider; + WalletRepository = walletRepository; _explorerClientProvider = explorerClientProvider; _jsonSerializerSettings = jsonSerializerSettings; _dbContextFactory = dbContextFactory; - _eventAggregator = eventAggregator; _notificationSender = notificationSender; this.Logs = logs; } @@ -426,13 +428,10 @@ public class BitcoinLikePayoutHandler : IPayoutHandler if (isInternal) { payout.State = PayoutState.InProgress; - var walletId = new WalletId(payout.StoreDataId, newTransaction.CryptoCode); - _eventAggregator.Publish(new UpdateTransactionLabel(walletId, + await WalletRepository.AddWalletTransactionAttachment( + new WalletId(payout.StoreDataId, newTransaction.CryptoCode), newTransaction.NewTransactionEvent.TransactionData.TransactionHash, - UpdateTransactionLabel.PayoutTemplate(new () - { - {payout.PullPaymentDataId?? "", new List{payout.Id}} - }, walletId.ToString()))); + Attachment.Payout(payout.PullPaymentDataId, payout.Id)); } else { diff --git a/BTCPayServer/Data/WalletDataExtensions.cs b/BTCPayServer/Data/WalletDataExtensions.cs deleted file mode 100644 index f1645b5d1..000000000 --- a/BTCPayServer/Data/WalletDataExtensions.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System; -using Newtonsoft.Json; - -namespace BTCPayServer.Data -{ - public static class WalletDataExtensions - { - public static WalletBlobInfo GetBlobInfo(this WalletData walletData) - { - if (walletData.Blob == null || walletData.Blob.Length == 0) - { - return new WalletBlobInfo(); - } - var blobInfo = JsonConvert.DeserializeObject(ZipUtils.Unzip(walletData.Blob)); - return blobInfo; - } - public static void SetBlobInfo(this WalletData walletData, WalletBlobInfo blobInfo) - { - if (blobInfo == null) - { - walletData.Blob = Array.Empty(); - return; - } - walletData.Blob = ZipUtils.Zip(JsonConvert.SerializeObject(blobInfo)); - } - } -} diff --git a/BTCPayServer/Data/WalletTransactionDataExtensions.cs b/BTCPayServer/Data/WalletTransactionDataExtensions.cs deleted file mode 100644 index 7deb96d5f..000000000 --- a/BTCPayServer/Data/WalletTransactionDataExtensions.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using BTCPayServer.Client.Models; -using BTCPayServer.Services.Labels; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using Newtonsoft.Json.Serialization; - -namespace BTCPayServer.Data -{ - public class WalletTransactionInfo - { - public string Comment { get; set; } = string.Empty; - [JsonIgnore] - public Dictionary Labels { get; set; } = new Dictionary(); - } - public static class WalletTransactionDataExtensions - { - public static int MaxCommentSize = 200; - - public static WalletTransactionInfo GetBlobInfo(this WalletTransactionData walletTransactionData) - { - WalletTransactionInfo blobInfo; - if (walletTransactionData.Blob == null || walletTransactionData.Blob.Length == 0) - blobInfo = new WalletTransactionInfo(); - else - blobInfo = JsonConvert.DeserializeObject(ZipUtils.Unzip(walletTransactionData.Blob)); - if (!string.IsNullOrEmpty(walletTransactionData.Labels)) - { - if (walletTransactionData.Labels.StartsWith('[')) - { - foreach (var jtoken in JArray.Parse(walletTransactionData.Labels)) - { - var l = jtoken.Type == JTokenType.String ? Label.Parse(jtoken.Value()) - : Label.Parse(jtoken.ToString()); - blobInfo.Labels.TryAdd(l.Text, l); - } - } - else - { - // Legacy path - foreach (var token in walletTransactionData.Labels.Split(',', - StringSplitOptions.RemoveEmptyEntries)) - { - var l = Label.Parse(token); - blobInfo.Labels.TryAdd(l.Text, l); - } - } - } - return blobInfo; - } - static JsonSerializerSettings LabelSerializerSettings = new JsonSerializerSettings() - { - ContractResolver = new CamelCasePropertyNamesContractResolver(), - Formatting = Formatting.None - }; - public static void SetBlobInfo(this WalletTransactionData walletTransactionData, WalletTransactionInfo blobInfo) - { - if (blobInfo == null) - { - walletTransactionData.Labels = string.Empty; - walletTransactionData.Blob = Array.Empty(); - return; - } - walletTransactionData.Labels = new JArray( - blobInfo.Labels.Select(l => JsonConvert.SerializeObject(l.Value, LabelSerializerSettings)) - .Select(l => JObject.Parse(l)) - .OfType() - .ToArray()).ToString(); - walletTransactionData.Blob = ZipUtils.Zip(JsonConvert.SerializeObject(blobInfo)); - } - } -} diff --git a/BTCPayServer/Data/WalletTransactionInfo.cs b/BTCPayServer/Data/WalletTransactionInfo.cs new file mode 100644 index 000000000..779d89fc7 --- /dev/null +++ b/BTCPayServer/Data/WalletTransactionInfo.cs @@ -0,0 +1,87 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using BTCPayServer.Client.Models; +using BTCPayServer.Services; +using BTCPayServer.Services.Labels; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Serialization; + +namespace BTCPayServer.Data +{ + public class WalletTransactionInfo + { + + public WalletTransactionInfo(WalletId walletId) + { + WalletId = walletId; + } + [JsonIgnore] + public WalletId WalletId { get; } + public string Comment { get; set; } = string.Empty; + [JsonIgnore] + public List Attachments { get; set; } = new List(); + + [JsonIgnore] + public Dictionary LabelColors { get; set; } = new Dictionary(); + + [Obsolete] + Dictionary? _LegacyLabels; + [JsonIgnore] + [Obsolete] + public Dictionary LegacyLabels + { + get + { + if (_LegacyLabels is null) + { + var legacyLabels = new Dictionary(); + foreach (var tag in Attachments) + { + switch (tag.Type) + { + case "payout": + PayoutLabel legacyPayoutLabel; + if (legacyLabels.TryGetValue(tag.Type, out var existing) && + existing is PayoutLabel) + { + legacyPayoutLabel = (PayoutLabel)existing; + } + else + { + legacyPayoutLabel = new PayoutLabel(); + legacyLabels.Add(tag.Type, legacyPayoutLabel); + } + var ppid = tag.Data?["pullPaymentId"]?.Value() ?? ""; + if (!legacyPayoutLabel.PullPaymentPayouts.TryGetValue(ppid, out var payouts)) + { + payouts = new List(); + legacyPayoutLabel.PullPaymentPayouts.Add(ppid, payouts); + } + payouts.Add(tag.Id); + break; + case "payjoin": + case "payment-request": + case "app": + case "pj-exposed": + case "invoice": + legacyLabels.TryAdd(tag.Type, new ReferenceLabel(tag.Type, tag.Id)); + break; + default: + continue; + } + } + foreach (var label in LabelColors) + { + legacyLabels.TryAdd(label.Key, new RawLabel(label.Key)); + } + _LegacyLabels = legacyLabels; + } + return _LegacyLabels; + } + } + + } +} diff --git a/BTCPayServer/HostedServices/DbMigrationsHostedService.cs b/BTCPayServer/HostedServices/DbMigrationsHostedService.cs index b9425587f..36cb1d149 100644 --- a/BTCPayServer/HostedServices/DbMigrationsHostedService.cs +++ b/BTCPayServer/HostedServices/DbMigrationsHostedService.cs @@ -2,15 +2,20 @@ using System; using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using System.Threading.Tasks; +using BTCPayServer.Client.Models; using BTCPayServer.Configuration; using BTCPayServer.Data; using BTCPayServer.Logging; using BTCPayServer.Services; using BTCPayServer.Services.Invoices; +using BTCPayServer.Services.Labels; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; namespace BTCPayServer.HostedServices { @@ -45,10 +50,235 @@ namespace BTCPayServer.HostedServices { await MigratedInvoiceTextSearchToDb(settings.MigratedInvoiceTextSearchPages ?? 0); } - - // Refresh settings since these operations may run for very long time + if (settings.MigratedTransactionLabels != int.MaxValue) + { + await MigratedTransactionLabels(settings.MigratedTransactionLabels ?? 0); + } } + +#pragma warning disable CS0612 // Type or member is obsolete + class LegacyWalletTransactionInfo + { + public string Comment { get; set; } = string.Empty; + [JsonIgnore] + public Dictionary Labels { get; set; } = new Dictionary(); + } + + static LegacyWalletTransactionInfo GetBlobInfo(WalletTransactionData walletTransactionData) + { + LegacyWalletTransactionInfo blobInfo; + if (walletTransactionData.Blob == null || walletTransactionData.Blob.Length == 0) + blobInfo = new LegacyWalletTransactionInfo(); + else + blobInfo = JsonConvert.DeserializeObject(ZipUtils.Unzip(walletTransactionData.Blob)); + if (!string.IsNullOrEmpty(walletTransactionData.Labels)) + { + if (walletTransactionData.Labels.StartsWith('[')) + { + foreach (var jtoken in JArray.Parse(walletTransactionData.Labels)) + { + var l = jtoken.Type == JTokenType.String ? Label.Parse(jtoken.Value()) + : Label.Parse(jtoken.ToString()); + blobInfo.Labels.TryAdd(l.Text, l); + } + } + else + { + // Legacy path + foreach (var token in walletTransactionData.Labels.Split(',', + StringSplitOptions.RemoveEmptyEntries)) + { + var l = Label.Parse(token); + blobInfo.Labels.TryAdd(l.Text, l); + } + } + } + return blobInfo; + } + + internal async Task MigratedTransactionLabels(int startFromOffset) + { + // Only of 1000, that's what EF does anyway under the hood by default + int batchCount = 1000; + int total = 0; + HashSet<(string WalletId, string LabelId)> existingLabels; + using (var db = _dbContextFactory.CreateContext()) + { + total = await db.WalletTransactions.CountAsync(); + existingLabels = (await ( + db.WalletObjects.AsNoTracking() + .Where(wo => wo.Type == WalletObjectData.Types.Label) + .Select(wl => new { wl.WalletId, wl.Id }) + .ToListAsync())) + .Select(o => (o.WalletId, o.Id)).ToHashSet(); + } + + + +next: +// var insertedObjectInDBContext +// Need to keep track of this hack, or then EF has a bug where he crash on the .Add and get internally +// corrupted. + var ifuckinghateentityframework = new HashSet<(string WalletId, string Type, string Id)>(); + using (var db = _dbContextFactory.CreateContext()) + { + Logs.PayServer.LogInformation($"Wallet transaction label importing transactions {startFromOffset}/{total}"); + var txs = await db.WalletTransactions + .OrderByDescending(wt => wt.WalletDataId).ThenBy(wt => wt.TransactionId) + .Skip(startFromOffset) + .Take(batchCount) + .ToArrayAsync(); + + foreach (var tx in txs) + { + // Same as above + var ifuckinghateentityframework2 = new HashSet<(string Type, string Id)>(); + var blob = GetBlobInfo(tx); + db.WalletObjects.Add(new Data.WalletObjectData() + { + WalletId = tx.WalletDataId, + Type = Data.WalletObjectData.Types.Tx, + Id = tx.TransactionId, + Data = string.IsNullOrEmpty(blob.Comment) ? null : new JObject() { ["comment"] = blob.Comment }.ToString() + }); + + foreach (var label in blob.Labels) + { + var labelId = label.Key; + if (labelId.StartsWith("{", StringComparison.OrdinalIgnoreCase)) + { + try + { + labelId = JObject.Parse(label.Key)["value"].Value(); + } + catch + { + } + } + if (!existingLabels.Contains((tx.WalletDataId, labelId))) + { + JObject labelData = new JObject(); + labelData.Add("color", "#000"); + db.WalletObjects.Add(new WalletObjectData() + { + WalletId = tx.WalletDataId, + Type = WalletObjectData.Types.Label, + Id = labelId, + Data = labelData.ToString() + }); + existingLabels.Add((tx.WalletDataId, labelId)); + } + if (ifuckinghateentityframework2.Add((Data.WalletObjectData.Types.Label, labelId))) + db.WalletObjectLinks.Add(new WalletObjectLinkData() + { + WalletId = tx.WalletDataId, + ChildType = Data.WalletObjectData.Types.Tx, + ChildId = tx.TransactionId, + ParentType = Data.WalletObjectData.Types.Label, + ParentId = labelId + }); + + if (label.Value is ReferenceLabel reflabel) + { + if (IsReferenceLabel(reflabel.Type)) + { + if (ifuckinghateentityframework.Add((tx.WalletDataId, reflabel.Type, reflabel.Reference ?? String.Empty))) + db.WalletObjects.Add(new WalletObjectData() + { + WalletId = tx.WalletDataId, + Type = reflabel.Type, + Id = reflabel.Reference ?? String.Empty + }); + + if (ifuckinghateentityframework2.Add((reflabel.Type, reflabel.Reference ?? String.Empty))) + db.WalletObjectLinks.Add(new WalletObjectLinkData() + { + WalletId = tx.WalletDataId, + ChildType = Data.WalletObjectData.Types.Tx, + ChildId = tx.TransactionId, + ParentType = reflabel.Type, + ParentId = reflabel.Reference ?? String.Empty + }); + } + } + else if (label.Value is PayoutLabel payoutLabel) + { + foreach (var pp in payoutLabel.PullPaymentPayouts) + { + foreach (var payout in pp.Value) + { + var payoutData = string.IsNullOrEmpty(pp.Key) ? null : new JObject() + { + ["pullPaymentId"] = pp.Key + }; + if (ifuckinghateentityframework.Add((tx.WalletDataId, "payout", payout))) + db.WalletObjects.Add(new WalletObjectData() + { + WalletId = tx.WalletDataId, + Type = "payout", + Id = payout, + Data = payoutData?.ToString() + }); + if (ifuckinghateentityframework2.Add(("payout", payout))) + db.WalletObjectLinks.Add(new WalletObjectLinkData() + { + WalletId = tx.WalletDataId, + ChildType = Data.WalletObjectData.Types.Tx, + ChildId = tx.TransactionId, + ParentType = "payout", + ParentId = payout + }); + } + } + } + } + } + int retry = 0; +retrySave: + try + { + await db.SaveChangesAsync(); + } + catch (DbUpdateException ex) when (retry < 10) + { + foreach (var entry in ex.Entries) + { + if (entry.Entity is WalletObjectData wo && (IsReferenceLabel(wo.Type) || wo.Type == "payout")) + { + await entry.ReloadAsync(); + } + } + retry++; + goto retrySave; + } + if (txs.Length < batchCount) + { + var settings = await _settingsRepository.GetSettingAsync(); + settings.MigratedTransactionLabels = int.MaxValue; + await _settingsRepository.UpdateSetting(settings); + Logs.PayServer.LogInformation($"Wallet transaction label successfully migrated"); + return; + } + else + { + startFromOffset += batchCount; + var settings = await _settingsRepository.GetSettingAsync(); + settings.MigratedTransactionLabels = startFromOffset; + await _settingsRepository.UpdateSetting(settings); + goto next; + } + } + } + + private static bool IsReferenceLabel(string type) + { + return type == "invoice" || + type == "payment-request" || + type == "app" || + type == "pj-exposed"; + } +#pragma warning restore CS0612 // Type or member is obsolete private async Task MigratedInvoiceTextSearchToDb(int startFromPage) { // deleting legacy DBriize database if present @@ -97,7 +327,7 @@ namespace BTCPayServer.HostedServices textSearch.Add(invoice.RefundMail); // TODO: Are there more things to cache? PaymentData? InvoiceRepository.AddToTextSearch(ctx, - new InvoiceData { Id = invoice.Id, InvoiceSearchData = new List() }, + new Data.InvoiceData { Id = invoice.Id, InvoiceSearchData = new List() }, textSearch.ToArray()); } diff --git a/BTCPayServer/HostedServices/TransactionLabelMarkerHostedService.cs b/BTCPayServer/HostedServices/TransactionLabelMarkerHostedService.cs index ac1fa9763..1704c04e5 100644 --- a/BTCPayServer/HostedServices/TransactionLabelMarkerHostedService.cs +++ b/BTCPayServer/HostedServices/TransactionLabelMarkerHostedService.cs @@ -1,3 +1,4 @@ +#nullable enable using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -14,6 +15,8 @@ using BTCPayServer.Services.Apps; using BTCPayServer.Services.Labels; using BTCPayServer.Services.PaymentRequests; using NBitcoin; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; namespace BTCPayServer.HostedServices { @@ -32,7 +35,6 @@ namespace BTCPayServer.HostedServices protected override void SubscribeToEvents() { Subscribe(); - Subscribe(); } protected override async Task ProcessEvent(object evt, CancellationToken cancellationToken) { @@ -42,136 +44,21 @@ namespace BTCPayServer.HostedServices { var walletId = new WalletId(invoiceEvent.Invoice.StoreId, invoiceEvent.Payment.GetCryptoCode()); var transactionId = bitcoinLikePaymentData.Outpoint.Hash; - var labels = new List<(string color, Label label)> + var labels = new List { - UpdateTransactionLabel.InvoiceLabelTemplate(invoiceEvent.Invoice.Id) + Attachment.Invoice(invoiceEvent.Invoice.Id) }; foreach (var paymentId in PaymentRequestRepository.GetPaymentIdsFromInternalTags(invoiceEvent.Invoice)) { - labels.Add(UpdateTransactionLabel.PaymentRequestLabelTemplate(paymentId)); + labels.Add(Attachment.PaymentRequest(paymentId)); } foreach (var appId in AppService.GetAppInternalTags(invoiceEvent.Invoice)) { - labels.Add(UpdateTransactionLabel.AppLabelTemplate(appId)); + labels.Add(Attachment.App(appId)); } - - - _eventAggregator.Publish(new UpdateTransactionLabel(walletId, transactionId, labels)); + await _walletRepository.AddWalletTransactionAttachment(walletId, transactionId, labels); } - else if (evt is UpdateTransactionLabel updateTransactionLabel) - { - var walletTransactionsInfo = - await _walletRepository.GetWalletTransactionsInfo(updateTransactionLabel.WalletId); - var walletBlobInfo = await _walletRepository.GetWalletInfo(updateTransactionLabel.WalletId); - await Task.WhenAll(updateTransactionLabel.TransactionLabels.Select(async pair => - { - var txId = pair.Key.ToString(); - var coloredLabels = pair.Value; - if (!walletTransactionsInfo.TryGetValue(txId, out var walletTransactionInfo)) - { - walletTransactionInfo = new WalletTransactionInfo(); - } - - bool walletNeedUpdate = false; - foreach (var cl in coloredLabels) - { - if (walletBlobInfo.LabelColors.TryGetValue(cl.label.Text, out var currentColor)) - { - if (currentColor != cl.color) - { - walletNeedUpdate = true; - walletBlobInfo.LabelColors[cl.label.Text] = currentColor; - } - } - else - { - walletNeedUpdate = true; - walletBlobInfo.LabelColors.AddOrReplace(cl.label.Text, cl.color); - } - } - - if (walletNeedUpdate) - await _walletRepository.SetWalletInfo(updateTransactionLabel.WalletId, walletBlobInfo); - foreach (var cl in coloredLabels) - { - var label = cl.label; - if (walletTransactionInfo.Labels.TryGetValue(label.Text, out var existingLabel)) - { - label = label.Merge(existingLabel); - } - - walletTransactionInfo.Labels.AddOrReplace(label.Text, label); - } - - await _walletRepository.SetWalletTransactionInfo(updateTransactionLabel.WalletId, - txId, walletTransactionInfo); - })); - } - } - } - - public class UpdateTransactionLabel - { - public UpdateTransactionLabel() - { - - } - public UpdateTransactionLabel(WalletId walletId, uint256 txId, (string color, Label label) colorLabel) - { - WalletId = walletId; - TransactionLabels = new Dictionary>(); - TransactionLabels.Add(txId, new List<(string color, Label label)>() { colorLabel }); - } - public UpdateTransactionLabel(WalletId walletId, uint256 txId, List<(string color, Label label)> colorLabels) - { - WalletId = walletId; - TransactionLabels = new Dictionary>(); - TransactionLabels.Add(txId, colorLabels); - } - public static (string color, Label label) PayjoinLabelTemplate() - { - return ("#51b13e", new RawLabel("payjoin")); - } - - public static (string color, Label label) InvoiceLabelTemplate(string invoice) - { - return ("#cedc21", new ReferenceLabel("invoice", invoice)); - } - public static (string color, Label label) PaymentRequestLabelTemplate(string paymentRequestId) - { - return ("#489D77", new ReferenceLabel("payment-request", paymentRequestId)); - } - public static (string color, Label label) AppLabelTemplate(string appId) - { - return ("#5093B6", new ReferenceLabel("app", appId)); - } - - public static (string color, Label label) PayjoinExposedLabelTemplate(string invoice) - { - return ("#51b13e", new ReferenceLabel("pj-exposed", invoice)); - } - - public static (string color, Label label) PayoutTemplate(Dictionary> pullPaymentToPayouts, string walletId) - { - return ("#3F88AF", new PayoutLabel() - { - PullPaymentPayouts = pullPaymentToPayouts, - WalletId = walletId - }); - } - public WalletId WalletId { get; set; } - public Dictionary> TransactionLabels { get; set; } - public override string ToString() - { - var result = new StringBuilder(); - foreach (var transactionLabel in TransactionLabels) - { - result.AppendLine(CultureInfo.InvariantCulture, - $"Adding {transactionLabel.Value.Count} labels to {transactionLabel.Key} in wallet {WalletId}"); - } - - return result.ToString(); } } } diff --git a/BTCPayServer/Hosting/BTCPayServerServices.cs b/BTCPayServer/Hosting/BTCPayServerServices.cs index 132738daf..ff07c07b4 100644 --- a/BTCPayServer/Hosting/BTCPayServerServices.cs +++ b/BTCPayServer/Hosting/BTCPayServerServices.cs @@ -103,7 +103,6 @@ namespace BTCPayServer.Hosting services.TryAddSingleton(); services.TryAddSingleton(provider => provider.GetService()); services.TryAddSingleton(provider => provider.GetService()); - services.TryAddSingleton(); services.TryAddSingleton(); services.AddSingleton(provider => provider.GetRequiredService()); services.AddSingleton(); diff --git a/BTCPayServer/Hosting/MigrationStartupTask.cs b/BTCPayServer/Hosting/MigrationStartupTask.cs index c3b5788b5..c5470963b 100644 --- a/BTCPayServer/Hosting/MigrationStartupTask.cs +++ b/BTCPayServer/Hosting/MigrationStartupTask.cs @@ -28,6 +28,7 @@ using Microsoft.Extensions.Options; using NBitcoin; using NBitcoin.DataEncoders; using NBXplorer; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using PeterO.Cbor; using PayoutData = BTCPayServer.Data.PayoutData; @@ -215,6 +216,12 @@ namespace BTCPayServer.Hosting settings.MigrateEmailServerDisableTLSCerts = true; await _Settings.UpdateSetting(settings); } + if (!settings.MigrateWalletColors) + { + await MigrateMigrateLabels(); + settings.MigrateWalletColors = true; + await _Settings.UpdateSetting(settings); + } } catch (Exception ex) { @@ -223,6 +230,59 @@ namespace BTCPayServer.Hosting } } +#pragma warning disable CS0612 // Type or member is obsolete + + static WalletBlobInfo GetBlobInfo(WalletData walletData) + { + if (walletData.Blob == null || walletData.Blob.Length == 0) + { + return new WalletBlobInfo(); + } + var blobInfo = JsonConvert.DeserializeObject(ZipUtils.Unzip(walletData.Blob)); + return blobInfo; + } + + private async Task MigrateMigrateLabels() + { + await using var ctx = _DBContextFactory.CreateContext(); + var wallets = await ctx.Wallets.AsNoTracking().ToArrayAsync(); + foreach (var wallet in wallets) + { + var blob = GetBlobInfo(wallet); + HashSet labels = new HashSet(blob.LabelColors.Count); + foreach (var label in blob.LabelColors) + { + var labelId = label.Key; + if (labelId.StartsWith("{", StringComparison.OrdinalIgnoreCase)) + { + try + { + labelId = JObject.Parse(label.Key)["value"].Value(); + } + catch + { + } + } + if (!labels.Add(labelId)) + continue; + var obj = new JObject(); + obj.Add("color", label.Value); + var labelObjId = new WalletObjectId(WalletId.Parse(wallet.Id), + WalletObjectData.Types.Label, + labelId); + ctx.WalletObjects.Add(new WalletObjectData() + { + WalletId = wallet.Id, + Type = WalletObjectData.Types.Label, + Id = labelId, + Data = obj.ToString() + }); + } + } + await ctx.SaveChangesAsync(); + } +#pragma warning restore CS0612 // Type or member is obsolete + // In the past, if a server was considered local network, then we would disable TLS checks. // Now we don't do it anymore, as we have an explicit flag (DisableCertificateCheck) to control the behavior. // But we need to migrate old users that relied on the behavior before. diff --git a/BTCPayServer/Models/WalletViewModels/ListTransactionsViewModel.cs b/BTCPayServer/Models/WalletViewModels/ListTransactionsViewModel.cs index 462c02513..7309f8100 100644 --- a/BTCPayServer/Models/WalletViewModels/ListTransactionsViewModel.cs +++ b/BTCPayServer/Models/WalletViewModels/ListTransactionsViewModel.cs @@ -15,9 +15,9 @@ namespace BTCPayServer.Models.WalletViewModels public string Link { get; set; } public bool Positive { get; set; } public string Balance { get; set; } - public HashSet Labels { get; set; } = new HashSet(); + public HashSet Tags { get; set; } = new HashSet(); } - public HashSet Labels { get; set; } = new HashSet(); + public HashSet<(string Text, string Color, string TextColor)> Labels { get; set; } = new HashSet<(string Text, string Color, string TextColor)>(); public List Transactions { get; set; } = new List(); public override int CurrentPageCount => Transactions.Count; public string CryptoCode { get; set; } diff --git a/BTCPayServer/Services/Labels/ColoredLabel.cs b/BTCPayServer/Models/WalletViewModels/TransactionTagModel.cs similarity index 68% rename from BTCPayServer/Services/Labels/ColoredLabel.cs rename to BTCPayServer/Models/WalletViewModels/TransactionTagModel.cs index c4936a866..ec0741d9c 100644 --- a/BTCPayServer/Services/Labels/ColoredLabel.cs +++ b/BTCPayServer/Models/WalletViewModels/TransactionTagModel.cs @@ -1,28 +1,24 @@ using System; -namespace BTCPayServer.Services.Labels +namespace BTCPayServer.Models.WalletViewModels { - public class ColoredLabel + public class TransactionTagModel { - internal ColoredLabel() - { - } - public string Text { get; internal set; } public string Color { get; internal set; } public string TextColor { get; internal set; } public string Link { get; internal set; } - public string Tooltip { get; internal set; } + public string Tooltip { get; internal set; } = String.Empty; public override bool Equals(object obj) { - ColoredLabel item = obj as ColoredLabel; + TransactionTagModel item = obj as TransactionTagModel; if (item == null) return false; return Text.Equals(item.Text, StringComparison.OrdinalIgnoreCase); } - public static bool operator ==(ColoredLabel a, ColoredLabel b) + public static bool operator ==(TransactionTagModel a, TransactionTagModel b) { if (System.Object.ReferenceEquals(a, b)) return true; @@ -31,7 +27,7 @@ namespace BTCPayServer.Services.Labels return a.Text == b.Text; } - public static bool operator !=(ColoredLabel a, ColoredLabel b) + public static bool operator !=(TransactionTagModel a, TransactionTagModel b) { return !(a == b); } diff --git a/BTCPayServer/Models/WalletViewModels/WalletSendModel.cs b/BTCPayServer/Models/WalletViewModels/WalletSendModel.cs index f3b385779..044b7f0ba 100644 --- a/BTCPayServer/Models/WalletViewModels/WalletSendModel.cs +++ b/BTCPayServer/Models/WalletViewModels/WalletSendModel.cs @@ -73,7 +73,7 @@ namespace BTCPayServer.Models.WalletViewModels public class InputSelectionOption { - public IEnumerable Labels { get; set; } + public IEnumerable Labels { get; set; } public string Comment { get; set; } public decimal Amount { get; set; } public string Outpoint { get; set; } diff --git a/BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs b/BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs index f33789b54..d944925f7 100644 --- a/BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs +++ b/BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs @@ -81,6 +81,7 @@ namespace BTCPayServer.Payments.PayJoin } private readonly BTCPayNetworkProvider _btcPayNetworkProvider; private readonly InvoiceRepository _invoiceRepository; + private readonly WalletRepository _walletRepository; private readonly ExplorerClientProvider _explorerClientProvider; private readonly BTCPayWalletProvider _btcPayWalletProvider; private readonly UTXOLocker _utxoLocker; @@ -96,6 +97,7 @@ namespace BTCPayServer.Payments.PayJoin public PayJoinEndpointController(BTCPayNetworkProvider btcPayNetworkProvider, InvoiceRepository invoiceRepository, ExplorerClientProvider explorerClientProvider, + WalletRepository walletRepository, BTCPayWalletProvider btcPayWalletProvider, UTXOLocker utxoLocker, EventAggregator eventAggregator, @@ -109,6 +111,7 @@ namespace BTCPayServer.Payments.PayJoin { _btcPayNetworkProvider = btcPayNetworkProvider; _invoiceRepository = invoiceRepository; + _walletRepository = walletRepository; _explorerClientProvider = explorerClientProvider; _btcPayWalletProvider = btcPayWalletProvider; _utxoLocker = utxoLocker; @@ -503,23 +506,14 @@ namespace BTCPayServer.Payments.PayJoin } await _btcPayWalletProvider.GetWallet(network).SaveOffchainTransactionAsync(ctx.OriginalTransaction); - var labels = selectedUTXOs.GroupBy(pair => pair.Key.Hash).Select(utxo => - new KeyValuePair>(utxo.Key, - new List<(string color, Label label)>() - { - UpdateTransactionLabel.PayjoinExposedLabelTemplate(invoice?.Id) - })) - .ToDictionary(pair => pair.Key, pair => pair.Value); + + foreach (var utxo in selectedUTXOs) + { + await _walletRepository.AddWalletTransactionAttachment(walletId, utxo.Key.Hash, Attachment.PayjoinExposed(invoice?.Id)); + } + await _walletRepository.AddWalletTransactionAttachment(walletId, originalPaymentData.PayjoinInformation.CoinjoinTransactionHash, Attachment.Payjoin()); + - labels.Add(originalPaymentData.PayjoinInformation.CoinjoinTransactionHash, new List<(string color, Label label)>() - { - UpdateTransactionLabel.PayjoinLabelTemplate() - }); - _eventAggregator.Publish(new UpdateTransactionLabel() - { - WalletId = walletId, - TransactionLabels = labels - }); ctx.Success(); // BTCPay Server support PSBT set as hex if (psbtFormat && HexEncoder.IsWellFormed(rawBody)) diff --git a/BTCPayServer/PayoutProcessors/OnChain/OnChainAutomatedPayoutProcessor.cs b/BTCPayServer/PayoutProcessors/OnChain/OnChainAutomatedPayoutProcessor.cs index 4789dc891..705f0e0e3 100644 --- a/BTCPayServer/PayoutProcessors/OnChain/OnChainAutomatedPayoutProcessor.cs +++ b/BTCPayServer/PayoutProcessors/OnChain/OnChainAutomatedPayoutProcessor.cs @@ -39,6 +39,7 @@ namespace BTCPayServer.PayoutProcessors.OnChain ILoggerFactory logger, BitcoinLikePayoutHandler bitcoinLikePayoutHandler, EventAggregator eventAggregator, + WalletRepository walletRepository, StoreRepository storeRepository, PayoutProcessorData payoutProcesserSettings, PullPaymentHostedService pullPaymentHostedService, @@ -51,8 +52,11 @@ namespace BTCPayServer.PayoutProcessors.OnChain _btcPayNetworkJsonSerializerSettings = btcPayNetworkJsonSerializerSettings; _bitcoinLikePayoutHandler = bitcoinLikePayoutHandler; _eventAggregator = eventAggregator; + WalletRepository = walletRepository; } + public WalletRepository WalletRepository { get; } + protected override async Task Process(ISupportedPaymentMethod paymentMethod, List payouts) { var storePaymentMethod = paymentMethod as DerivationSchemeSettings; @@ -171,12 +175,9 @@ namespace BTCPayServer.PayoutProcessors.OnChain var walletId = new WalletId(_PayoutProcesserSettings.StoreId, PaymentMethodId.CryptoCode); foreach (PayoutData payoutData in transfersProcessing) { - _eventAggregator.Publish(new UpdateTransactionLabel(walletId, + await WalletRepository.AddWalletTransactionAttachment(walletId, txHash, - UpdateTransactionLabel.PayoutTemplate(new () - { - {payoutData.PullPaymentDataId?? "", new List{payoutData.Id}} - }, walletId.ToString()))); + Attachment.Payout(payoutData.PullPaymentDataId, payoutData.Id)); } await Task.WhenAny(tcs.Task, task); } diff --git a/BTCPayServer/Services/Attachment.cs b/BTCPayServer/Services/Attachment.cs new file mode 100644 index 000000000..d6d771ce5 --- /dev/null +++ b/BTCPayServer/Services/Attachment.cs @@ -0,0 +1,59 @@ +#nullable enable +using System.Collections.Generic; +using BTCPayServer.Client.Models; +using BTCPayServer.Services.Labels; +using Newtonsoft.Json.Linq; + +namespace BTCPayServer.Services +{ + public class Attachment + { + public string Type { get; } + public string Id { get; } + public JObject? Data { get; } + + public Attachment(string type, string? id = null, JObject? data = null) + { + Type = type; + Id = id ?? string.Empty; + Data = data; + } + public static Attachment Payjoin() + { + return new Attachment("payjoin"); + } + public static Attachment Invoice(string invoice) + { + return new Attachment("invoice", invoice); + } + public static Attachment PaymentRequest(string paymentRequestId) + { + return new Attachment("payment-request", paymentRequestId); + } + public static Attachment App(string appId) + { + return new Attachment("app", appId); + } + + public static Attachment PayjoinExposed(string? invoice) + { + return new Attachment("pj-exposed", invoice); + } + + public static IEnumerable Payout(string? pullPaymentId, string payoutId) + { + if (string.IsNullOrEmpty(pullPaymentId)) + { + yield return new Attachment("payout", payoutId); + } + else + { + yield return new Attachment("payout", payoutId, new JObject() + { + ["pullPaymentId"] = pullPaymentId + }); + yield return new Attachment("pull-payment", pullPaymentId); + } + } + } +} diff --git a/BTCPayServer/Services/Labels/Label.cs b/BTCPayServer/Services/Labels/Label.cs index 0fd5667e3..a3765c0f9 100644 --- a/BTCPayServer/Services/Labels/Label.cs +++ b/BTCPayServer/Services/Labels/Label.cs @@ -9,13 +9,9 @@ using Newtonsoft.Json.Linq; namespace BTCPayServer.Services.Labels { + [Obsolete] public abstract class Label : LabelData { - public virtual Label Merge(LabelData other) - { - return this; - } - static void FixLegacy(JObject jObj, ReferenceLabel refLabel) { if (refLabel.Reference is null && jObj.ContainsKey("id")) @@ -49,6 +45,18 @@ namespace BTCPayServer.Services.Labels rawLabel.Type = "raw"; FixLegacy(jObj, (Label)rawLabel); } + + static Label() + { + SerializerSettings = new JsonSerializerSettings() + { + ContractResolver = new Newtonsoft.Json.Serialization.CamelCasePropertyNamesContractResolver() + }; + Serializer = JsonSerializer.Create(SerializerSettings); + } + + public static JsonSerializerSettings SerializerSettings; + public static JsonSerializer Serializer; public static Label Parse(string str) { ArgumentNullException.ThrowIfNull(str); @@ -95,6 +103,7 @@ namespace BTCPayServer.Services.Labels } } + [Obsolete] public class RawLabel : Label { public RawLabel() @@ -106,6 +115,7 @@ namespace BTCPayServer.Services.Labels Text = text; } } + [Obsolete] public class ReferenceLabel : Label { public ReferenceLabel() @@ -121,6 +131,7 @@ namespace BTCPayServer.Services.Labels [JsonProperty("ref")] public string Reference { get; set; } } + [Obsolete] public class PayoutLabel : Label { public PayoutLabel() @@ -130,21 +141,5 @@ namespace BTCPayServer.Services.Labels } public Dictionary> PullPaymentPayouts { get; set; } = new(); - public string WalletId { get; set; } - - public override Label Merge(LabelData other) - { - if (other is not PayoutLabel otherPayoutLabel) return base.Merge(other); - foreach (var pullPaymentPayout in otherPayoutLabel.PullPaymentPayouts) - { - if (!PullPaymentPayouts.TryGetValue(pullPaymentPayout.Key, out var pullPaymentPayouts)) - { - pullPaymentPayouts = new List(); - PullPaymentPayouts.Add(pullPaymentPayout.Key, pullPaymentPayouts); - } - pullPaymentPayouts.AddRange(pullPaymentPayout.Value); - } - return base.Merge(other); - } } } diff --git a/BTCPayServer/Services/Labels/LabelFactory.cs b/BTCPayServer/Services/Labels/LabelFactory.cs deleted file mode 100644 index 11a6e35a2..000000000 --- a/BTCPayServer/Services/Labels/LabelFactory.cs +++ /dev/null @@ -1,197 +0,0 @@ -#nullable enable -using System; -using System.Drawing; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Routing; -using BTCPayServer.Data; -using BTCPayServer.Client.Models; -using BTCPayServer.Abstractions.Extensions; - -namespace BTCPayServer.Services.Labels -{ - public class LabelFactory - { - private readonly LinkGenerator _linkGenerator; - private readonly WalletRepository _walletRepository; - - public LabelFactory( - LinkGenerator linkGenerator, - WalletRepository walletRepository - ) - { - _linkGenerator = linkGenerator; - _walletRepository = walletRepository; - } - - public IEnumerable ColorizeTransactionLabels(WalletBlobInfo walletBlobInfo, WalletTransactionInfo transactionInfo, - HttpRequest request) - { - foreach (var label in transactionInfo.Labels) - { - walletBlobInfo.LabelColors.TryGetValue(label.Value.Text, out var color); - yield return CreateLabel(label.Value, color, request); - } - } - - public IEnumerable GetWalletColoredLabels(WalletBlobInfo walletBlobInfo, HttpRequest request) - { - foreach (var kv in walletBlobInfo.LabelColors) - { - yield return CreateLabel(new RawLabel() { Text = kv.Key }, kv.Value, request); - } - } - - const string DefaultColor = "#000"; - private ColoredLabel CreateLabel(LabelData uncoloredLabel, string? color, HttpRequest request) - { - ArgumentNullException.ThrowIfNull(uncoloredLabel); - color ??= DefaultColor; - - ColoredLabel coloredLabel = new ColoredLabel - { - Text = uncoloredLabel.Text, - Color = color, - Tooltip = "", - TextColor = TextColor(color) - }; - - string PayoutLabelText(KeyValuePair>? pair = null) - { - if (pair is null) - { - return "Paid a payout"; - } - return pair.Value.Value.Count == 1 ? $"Paid a payout {(string.IsNullOrEmpty(pair.Value.Key)? string.Empty: $"of a pull payment ({pair.Value.Key})")}" : $"Paid {pair.Value.Value.Count} payouts {(string.IsNullOrEmpty(pair.Value.Key)? string.Empty: $"of a pull payment ({pair.Value.Key})")}"; - } - - if (uncoloredLabel is ReferenceLabel refLabel) - { - var refInLabel = string.IsNullOrEmpty(refLabel.Reference) ? string.Empty : $"({refLabel.Reference})"; - switch (uncoloredLabel.Type) - { - case "invoice": - coloredLabel.Tooltip = $"Received through an invoice {refInLabel}"; - coloredLabel.Link = string.IsNullOrEmpty(refLabel.Reference) - ? null - : _linkGenerator.InvoiceLink(refLabel.Reference, request.Scheme, request.Host, request.PathBase); - break; - case "payment-request": - coloredLabel.Tooltip = $"Received through a payment request {refInLabel}"; - coloredLabel.Link = string.IsNullOrEmpty(refLabel.Reference) - ? null - : _linkGenerator.PaymentRequestLink(refLabel.Reference, request.Scheme, request.Host, request.PathBase); - break; - case "app": - coloredLabel.Tooltip = $"Received through an app {refInLabel}"; - coloredLabel.Link = string.IsNullOrEmpty(refLabel.Reference) - ? null - : _linkGenerator.AppLink(refLabel.Reference, request.Scheme, request.Host, request.PathBase); - break; - case "pj-exposed": - coloredLabel.Tooltip = $"This UTXO was exposed through a PayJoin proposal for an invoice {refInLabel}"; - coloredLabel.Link = string.IsNullOrEmpty(refLabel.Reference) - ? null - : _linkGenerator.InvoiceLink(refLabel.Reference, request.Scheme, request.Host, request.PathBase); - break; - } - } - else if (uncoloredLabel is PayoutLabel payoutLabel) - { - coloredLabel.Tooltip = payoutLabel.PullPaymentPayouts?.Count switch - { - null => PayoutLabelText(), - 0 => PayoutLabelText(), - 1 => PayoutLabelText(payoutLabel.PullPaymentPayouts.First()), - _ => - $"
    {string.Join(string.Empty, payoutLabel.PullPaymentPayouts.Select(pair => $"
  • {PayoutLabelText(pair)}
    • "))}
" - }; - - coloredLabel.Link = string.IsNullOrEmpty(payoutLabel.WalletId) - ? null - : _linkGenerator.PayoutLink(payoutLabel.WalletId, null, PayoutState.Completed, request.Scheme, request.Host, - request.PathBase); - } - else if (uncoloredLabel.Text == "payjoin") - { - coloredLabel.Tooltip = $"This UTXO was part of a PayJoin transaction."; - } - return coloredLabel; - } - - // Borrowed from https://github.com/ManageIQ/guides/blob/master/labels.md - readonly string[] LabelColorScheme = - { - "#fbca04", - "#0e8a16", - "#ff7619", - "#84b6eb", - "#5319e7", - "#cdcdcd", - "#cc317c", - }; - - readonly int MaxLabelSize = 20; - - async public Task BuildLabel( - WalletBlobInfo walletBlobInfo, - HttpRequest request, - WalletTransactionInfo walletTransactionInfo, - WalletId walletId, - string transactionId, - string label - ) - { - label = label.Trim().TrimStart('{').ToLowerInvariant().Replace(',', ' ').Truncate(MaxLabelSize); - var labels = GetWalletColoredLabels(walletBlobInfo, request); - - if (!labels.Any(l => l.Text.Equals(label, StringComparison.OrdinalIgnoreCase))) - { - var chosenColor = ChooseBackgroundColor(walletBlobInfo, request); - walletBlobInfo.LabelColors.Add(label, chosenColor); - await _walletRepository.SetWalletInfo(walletId, walletBlobInfo); - } - - return new RawLabel(label); - } - - private string ChooseBackgroundColor( - WalletBlobInfo walletBlobInfo, - HttpRequest request - ) - { - var labels = GetWalletColoredLabels(walletBlobInfo, request); - - List allColors = new List(); - allColors.AddRange(LabelColorScheme); - allColors.AddRange(labels.Select(l => l.Color)); - var chosenColor = - allColors - .GroupBy(k => k) - .OrderBy(k => k.Count()) - .ThenBy(k => - { - var indexInColorScheme = Array.IndexOf(LabelColorScheme, k.Key); - - // Ensures that any label color which may not be in our label color scheme is given the least priority - return indexInColorScheme == -1 ? double.PositiveInfinity : indexInColorScheme; - }) - .First().Key; - - return chosenColor; - } - - private string TextColor(string bgColor) - { - int nThreshold = 105; - var bg = ColorTranslator.FromHtml(bgColor); - int bgDelta = Convert.ToInt32((bg.R * 0.299) + (bg.G * 0.587) + (bg.B * 0.114)); - Color color = (255 - bgDelta < nThreshold) ? Color.Black : Color.White; - return ColorTranslator.ToHtml(color); - } - } -} diff --git a/BTCPayServer/Services/MigrationSettings.cs b/BTCPayServer/Services/MigrationSettings.cs index f4ac52d82..e894f00a3 100644 --- a/BTCPayServer/Services/MigrationSettings.cs +++ b/BTCPayServer/Services/MigrationSettings.cs @@ -25,6 +25,7 @@ namespace BTCPayServer.Services // Done in DbMigrationsHostedService public int? MigratedInvoiceTextSearchPages { get; set; } + public int? MigratedTransactionLabels { get; set; } public bool MigrateAppCustomOption { get; set; } public bool MigratePayoutDestinationId { get; set; } public bool AddInitialUserBlob { get; set; } @@ -32,5 +33,6 @@ namespace BTCPayServer.Services public bool LighingAddressDatabaseMigration { get; set; } public bool AddStoreToPayout { get; set; } public bool MigrateEmailServerDisableTLSCerts { get; set; } + public bool MigrateWalletColors { get; set; } } } diff --git a/BTCPayServer/Services/WalletRepository.cs b/BTCPayServer/Services/WalletRepository.cs index 80f3814a7..0dd064cf9 100644 --- a/BTCPayServer/Services/WalletRepository.cs +++ b/BTCPayServer/Services/WalletRepository.cs @@ -1,12 +1,22 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading.Tasks; +using BTCPayServer.Abstractions.Extensions; using BTCPayServer.Data; +using BTCPayServer.HostedServices; +using BTCPayServer.Services.Labels; using Microsoft.EntityFrameworkCore; +using NBitcoin; +using NBitcoin.Crypto; +using Newtonsoft.Json.Linq; namespace BTCPayServer.Services { +#nullable enable + public record WalletObjectId(WalletId WalletId, string Type, string Id); +#nullable restore public class WalletRepository { private readonly ApplicationDbContextFactory _ContextFactory; @@ -16,74 +26,256 @@ namespace BTCPayServer.Services _ContextFactory = contextFactory ?? throw new ArgumentNullException(nameof(contextFactory)); } - public async Task SetWalletInfo(WalletId walletId, WalletBlobInfo blob) - { - ArgumentNullException.ThrowIfNull(walletId); - using var ctx = _ContextFactory.CreateContext(); - var walletData = new WalletData() { Id = walletId.ToString() }; - walletData.SetBlobInfo(blob); - var entity = await ctx.Wallets.AddAsync(walletData); - entity.State = EntityState.Modified; - try - { - await ctx.SaveChangesAsync(); - } - catch (DbUpdateException) // Does not exists - { - entity.State = EntityState.Added; - await ctx.SaveChangesAsync(); - } - } - public async Task> GetWalletTransactionsInfo(WalletId walletId, string[] transactionIds = null) { ArgumentNullException.ThrowIfNull(walletId); using var ctx = _ContextFactory.CreateContext(); - return (await ctx.WalletTransactions - .Where(w => w.WalletDataId == walletId.ToString()) - .Where(data => transactionIds == null || transactionIds.Contains(data.TransactionId)) - .Select(w => w) - .ToArrayAsync()) - .ToDictionary(w => w.TransactionId, w => w.GetBlobInfo()); + + + IQueryable wols; + IQueryable wos; + + // If we are using postgres, the `transactionIds.Contains(w.ChildId)` result in a long query like `ANY(@txId1, @txId2, @txId3, @txId4)` + // Such request isn't well optimized by postgres, and create different requests clogging up + // pg_stat_statements output, making it impossible to analyze the performance impact of this query. + if (ctx.Database.IsNpgsql() && transactionIds is not null) + { + wos = ctx.WalletObjects + .FromSqlInterpolated($"SELECT wos.* FROM unnest({transactionIds}) t JOIN \"WalletObjects\" wos ON wos.\"WalletId\"={walletId.ToString()} AND wos.\"Type\"={WalletObjectData.Types.Tx} AND wos.\"Id\"=t") + .AsNoTracking(); + wols = ctx.WalletObjectLinks + .FromSqlInterpolated($"SELECT wol.* FROM unnest({transactionIds}) t JOIN \"WalletObjectLinks\" wol ON wol.\"WalletId\"={walletId.ToString()} AND wol.\"ChildType\"={WalletObjectData.Types.Tx} AND wol.\"ChildId\"=t") + .AsNoTracking(); + } + else // Unefficient path + { + wos = ctx.WalletObjects + .AsNoTracking() + .Where(w => w.WalletId == walletId.ToString() && w.Type == WalletObjectData.Types.Tx && (transactionIds == null || transactionIds.Contains(w.Id))); + wols = ctx.WalletObjectLinks + .AsNoTracking() + .Where(w => w.WalletId == walletId.ToString() && w.ChildType == WalletObjectData.Types.Tx && (transactionIds == null || transactionIds.Contains(w.ChildId))); + } + var links = await wols + .Select(tx => + new + { + TxId = tx.ChildId, + AssociatedDataId = tx.ParentId, + AssociatedDataType = tx.ParentType, + AssociatedData = tx.Parent.Data + }) + .ToArrayAsync(); + var objs = await wos + .Select(tx => + new + { + TxId = tx.Id, + Data = tx.Data + }) + .ToArrayAsync(); + + var result = new Dictionary(objs.Length); + foreach (var obj in objs) + { + var data = obj.Data is null ? null : JObject.Parse(obj.Data); + result.Add(obj.TxId, new WalletTransactionInfo(walletId) + { + Comment = data?["comment"]?.Value() + }); + } + + + foreach (var row in links) + { + JObject data = row.AssociatedData is null ? null : JObject.Parse(row.AssociatedData); + var info = result[row.TxId]; + + if (row.AssociatedDataType == WalletObjectData.Types.Label) + { + info.LabelColors.TryAdd(row.AssociatedDataId, data["color"]?.Value() ?? "#000"); + } + else + { + info.Attachments.Add(new Attachment(row.AssociatedDataType, row.AssociatedDataId, row.AssociatedData is null ? null : JObject.Parse(row.AssociatedData))); + } + } + return result; } - public async Task GetWalletInfo(WalletId walletId) +#nullable enable + + public async Task<(string Label, string Color)[]> GetWalletLabels(WalletId walletId) { - ArgumentNullException.ThrowIfNull(walletId); using var ctx = _ContextFactory.CreateContext(); - var data = await ctx.Wallets - .Where(w => w.Id == walletId.ToString()) - .Select(w => w) - .FirstOrDefaultAsync(); - return data?.GetBlobInfo() ?? new WalletBlobInfo(); + return (await ctx.WalletObjects + .AsNoTracking() + .Where(w => w.WalletId == walletId.ToString() && w.Type == WalletObjectData.Types.Label) + .Select(o => new { o.Id, o.Data }) + .ToArrayAsync()) + .Select(o => (o.Id, JObject.Parse(o.Data)["color"]!.Value()!)) + .ToArray(); } - public async Task SetWalletTransactionInfo(WalletId walletId, string transactionId, WalletTransactionInfo walletTransactionInfo) + public async Task EnsureWalletObjectLink(WalletObjectId parent, WalletObjectId child) { - ArgumentNullException.ThrowIfNull(walletId); - ArgumentNullException.ThrowIfNull(transactionId); using var ctx = _ContextFactory.CreateContext(); - var walletData = new WalletTransactionData() { WalletDataId = walletId.ToString(), TransactionId = transactionId }; - walletData.SetBlobInfo(walletTransactionInfo); - var entity = await ctx.WalletTransactions.AddAsync(walletData); - entity.State = EntityState.Modified; + var l = new WalletObjectLinkData() + { + WalletId = parent.WalletId.ToString(), + ChildType = child.Type, + ChildId = child.Id, + ParentType = parent.Type, + ParentId = parent.Id + }; + ctx.WalletObjectLinks.Add(l); try { await ctx.SaveChangesAsync(); } - catch (DbUpdateException) // Does not exists + catch (DbUpdateException) // already exists { - entity.State = EntityState.Added; + } + } + + public static int MaxCommentSize = 200; + public async Task SetWalletObjectComment(WalletObjectId id, string comment) + { + ArgumentNullException.ThrowIfNull(id); + ArgumentNullException.ThrowIfNull(comment); + if (!string.IsNullOrEmpty(comment)) + await ModifyWalletObjectData(id, (o) => o["comment"] = comment.Trim().Truncate(MaxCommentSize)); + else + await ModifyWalletObjectData(id, (o) => o.Remove("comment")); + } + + + static WalletObjectData NewWalletObjectData(WalletObjectId id, JObject? data = null) + { + return new WalletObjectData() + { + WalletId = id.WalletId.ToString(), + Type = id.Type, + Id = id.Id, + Data = data?.ToString() + }; + } + public async Task ModifyWalletObjectData(WalletObjectId id, Action modify) + { + ArgumentNullException.ThrowIfNull(id); + ArgumentNullException.ThrowIfNull(modify); + using var ctx = _ContextFactory.CreateContext(); + var obj = await ctx.WalletObjects.FindAsync(id.WalletId.ToString(), id.Type, id.Id); + if (obj is null) + { + obj = NewWalletObjectData(id); + ctx.WalletObjects.Add(obj); + } + var currentData = obj.Data is null ? new JObject() : JObject.Parse(obj.Data); + modify(currentData); + obj.Data = currentData.ToString(); + if (obj.Data == "{}") + obj.Data = null; + await ctx.SaveChangesAsync(); + } + + const int MaxLabelSize = 50; + public async Task AddWalletObjectLabels(WalletObjectId id, params string[] labels) + { + ArgumentNullException.ThrowIfNull(id); + await EnsureWalletObject(id); + foreach (var l in labels.Select(l => l.Trim().Truncate(MaxLabelSize))) + { + var labelObjId = new WalletObjectId(id.WalletId, WalletObjectData.Types.Label, l); + await EnsureWalletObject(labelObjId, new JObject() + { + ["color"] = ColorPalette.Default.DeterministicColor(l) + }); + await EnsureWalletObjectLink(labelObjId, id); + } + } + + public Task AddWalletTransactionAttachment(WalletId walletId, uint256 txId, Attachment attachment) + { + return AddWalletTransactionAttachment(walletId, txId, new[] { attachment }); + } + public async Task AddWalletTransactionAttachment(WalletId walletId, uint256 txId, IEnumerable attachments) + { + ArgumentNullException.ThrowIfNull(walletId); + ArgumentNullException.ThrowIfNull(txId); + var txObjId = new WalletObjectId(walletId, WalletObjectData.Types.Tx, txId.ToString()); + await EnsureWalletObject(txObjId); + foreach (var attachment in attachments) + { + var labelObjId = new WalletObjectId(walletId, WalletObjectData.Types.Label, attachment.Type); + await EnsureWalletObject(labelObjId, new JObject() + { + ["color"] = ColorPalette.Default.DeterministicColor(attachment.Type) + }); + await EnsureWalletObjectLink(labelObjId, txObjId); + if (attachment.Data is not null || attachment.Id.Length != 0) + { + var data = new WalletObjectId(walletId, attachment.Type, attachment.Id); + await EnsureWalletObject(data, attachment.Data); + await EnsureWalletObjectLink(data, txObjId); + } + } + } + public async Task RemoveWalletObjectLabels(WalletObjectId id, params string[] labels) + { + ArgumentNullException.ThrowIfNull(id); + foreach (var l in labels.Select(l => l.Trim())) + { + var labelObjId = new WalletObjectId(id.WalletId, WalletObjectData.Types.Label, l); + using var ctx = _ContextFactory.CreateContext(); + ctx.WalletObjectLinks.Remove(new WalletObjectLinkData() + { + WalletId = id.WalletId.ToString(), + ChildId = id.Id, + ChildType = id.Type, + ParentId = labelObjId.Id, + ParentType = labelObjId.Type + }); try { await ctx.SaveChangesAsync(); } - catch (DbUpdateException) // the Wallet does not exists in the DB + catch (DbUpdateException) // Already deleted, do nothing { - await SetWalletInfo(walletId, new WalletBlobInfo()); - await ctx.SaveChangesAsync(); } } } + + public async Task SetWalletObject(WalletObjectId id, JObject? data) + { + ArgumentNullException.ThrowIfNull(id); + using var ctx = _ContextFactory.CreateContext(); + var o = NewWalletObjectData(id, data); + ctx.WalletObjects.Add(o); + try + { + await ctx.SaveChangesAsync(); + } + catch (DbUpdateException) // already exists + { + ctx.Entry(o).State = EntityState.Modified; + await ctx.SaveChangesAsync(); + } + } + + public async Task EnsureWalletObject(WalletObjectId id, JObject? data = null) + { + ArgumentNullException.ThrowIfNull(id); + using var ctx = _ContextFactory.CreateContext(); + ctx.WalletObjects.Add(NewWalletObjectData(id, data)); + try + { + await ctx.SaveChangesAsync(); + } + catch (DbUpdateException) // already exists + { + } + } +#nullable restore } } diff --git a/BTCPayServer/Services/Wallets/Export/TransactionsExport.cs b/BTCPayServer/Services/Wallets/Export/TransactionsExport.cs index 183964f1a..1849c78e8 100644 --- a/BTCPayServer/Services/Wallets/Export/TransactionsExport.cs +++ b/BTCPayServer/Services/Wallets/Export/TransactionsExport.cs @@ -41,7 +41,7 @@ namespace BTCPayServer.Services.Wallets.Export if (_walletTransactionsInfo.TryGetValue(tx.TransactionId.ToString(), out var transactionInfo)) { - model.Labels = transactionInfo.Labels?.Select(l => l.Value.Text).ToList(); + model.Labels = transactionInfo.LabelColors?.Select(l => l.Key).ToList(); model.Comment = transactionInfo.Comment; } diff --git a/BTCPayServer/Views/UIWallets/_WalletTransactionsList.cshtml b/BTCPayServer/Views/UIWallets/_WalletTransactionsList.cshtml index 10cc920c2..4b035d222 100644 --- a/BTCPayServer/Views/UIWallets/_WalletTransactionsList.cshtml +++ b/BTCPayServer/Views/UIWallets/_WalletTransactionsList.cshtml @@ -10,10 +10,10 @@ @transaction.Timestamp.ToBrowserDate() - @if (transaction.Labels.Any()) + @if (transaction.Tags.Any()) {
- @foreach (var label in transaction.Labels) + @foreach (var label in transaction.Tags) {
@foreach (var label in Model.Labels) { - @if (transaction.Labels.Contains(label)) + @if (transaction.Tags.Any(l => l.Text == label.Text)) { } diff --git a/BTCPayServer/wwwroot/swagger/v1/swagger.template.stores-wallet.on-chain.json b/BTCPayServer/wwwroot/swagger/v1/swagger.template.stores-wallet.on-chain.json index efceb506f..fd7b82778 100644 --- a/BTCPayServer/wwwroot/swagger/v1/swagger.template.stores-wallet.on-chain.json +++ b/BTCPayServer/wwwroot/swagger/v1/swagger.template.stores-wallet.on-chain.json @@ -708,6 +708,7 @@ "LabelData": { "type": "object", "additionalProperties": true, + "deprecated": true, "properties": { "type": { "type": "string", @@ -762,6 +763,7 @@ }, "labels": { "description": "Labels linked to this transaction", + "deprecated": true, "type": "array", "items": { "$ref": "#/components/schemas/LabelData" @@ -810,6 +812,7 @@ }, "labels": { "description": "Labels linked to this transaction", + "deprecated": true, "type": "array", "items": { "$ref": "#/components/schemas/LabelData" @@ -904,6 +907,7 @@ }, "labels": { "nullable": true, + "deprecated": true, "description": "Transaction labels", "type": "array", "items": {