From 3cf1aa00fa5002432f5586531e3687d8406e5716 Mon Sep 17 00:00:00 2001 From: Nicolas Dorier Date: Mon, 23 Sep 2024 17:06:56 +0900 Subject: [PATCH] Payments should use composite key (#6240) * Payments should use composite key * Invert PK for InvoiceAddress --- BTCPayServer.Data/ApplicationDbContext.cs | 2 +- BTCPayServer.Data/BTCPayServer.Data.csproj | 1 + .../DBScripts/005.PaymentsRenaming.sql | 17 ++++++++ BTCPayServer.Data/Data/AddressInvoiceData.cs | 2 +- .../Data/PaymentData.Migration.cs | 6 +-- BTCPayServer.Data/Data/PaymentData.cs | 6 ++- .../20240919085726_refactorinvoiceaddress.cs | 4 +- .../20240923065254_refactorpayments.cs | 39 +++++++++++++++++++ ...ces.cs => 20240923071444_temprefactor2.cs} | 19 ++++----- .../ApplicationDbContextModelSnapshot.cs | 14 +++---- BTCPayServer.Tests/TestAccount.cs | 4 +- .../TestData/InvoiceMigrationTestVectors.json | 8 ++-- .../Controllers/UIInvoiceController.UI.cs | 2 +- BTCPayServer/Data/PaymentDataExtensions.cs | 4 +- .../Services/Invoices/InvoiceRepository.cs | 4 +- .../Services/Invoices/PaymentService.cs | 2 +- 16 files changed, 98 insertions(+), 36 deletions(-) create mode 100644 BTCPayServer.Data/DBScripts/005.PaymentsRenaming.sql create mode 100644 BTCPayServer.Data/Migrations/20240923065254_refactorpayments.cs rename BTCPayServer.Data/Migrations/{20240405052858_cleanup_address_invoices.cs => 20240923071444_temprefactor2.cs} (51%) diff --git a/BTCPayServer.Data/ApplicationDbContext.cs b/BTCPayServer.Data/ApplicationDbContext.cs index 38c3f6e6a..6bdb16718 100644 --- a/BTCPayServer.Data/ApplicationDbContext.cs +++ b/BTCPayServer.Data/ApplicationDbContext.cs @@ -82,7 +82,7 @@ namespace BTCPayServer.Data PairingCodeData.OnModelCreating(builder); //PayjoinLock.OnModelCreating(builder); PaymentRequestData.OnModelCreating(builder, Database); - PaymentData.OnModelCreating(builder, Database); + PaymentData.OnModelCreating(builder); PayoutData.OnModelCreating(builder, Database); //PlannedTransaction.OnModelCreating(builder); PullPaymentData.OnModelCreating(builder, Database); diff --git a/BTCPayServer.Data/BTCPayServer.Data.csproj b/BTCPayServer.Data/BTCPayServer.Data.csproj index 0a3c73e52..fdece06d5 100644 --- a/BTCPayServer.Data/BTCPayServer.Data.csproj +++ b/BTCPayServer.Data/BTCPayServer.Data.csproj @@ -22,5 +22,6 @@ + diff --git a/BTCPayServer.Data/DBScripts/005.PaymentsRenaming.sql b/BTCPayServer.Data/DBScripts/005.PaymentsRenaming.sql new file mode 100644 index 000000000..b52c5e742 --- /dev/null +++ b/BTCPayServer.Data/DBScripts/005.PaymentsRenaming.sql @@ -0,0 +1,17 @@ +DROP FUNCTION get_monitored_invoices; +CREATE OR REPLACE FUNCTION get_monitored_invoices(payment_method_id TEXT) +RETURNS TABLE (invoice_id TEXT, payment_id TEXT, payment_method_id TEXT) AS $$ +WITH cte AS ( +-- Get all the invoices which are pending. Even if no payments. +SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i LEFT JOIN "Payments" p ON i."Id" = p."InvoiceDataId" + WHERE is_pending(i."Status") +UNION ALL +-- For invoices not pending, take all of those which have pending payments +SELECT i."Id", p."Id", p."PaymentMethodId" payment_method_id FROM "Invoices" i INNER JOIN "Payments" p ON i."Id" = p."InvoiceDataId" + WHERE is_pending(p."Status") AND NOT is_pending(i."Status")) +SELECT cte.* FROM cte +LEFT JOIN "Payments" p ON cte.payment_id=p."Id" AND cte.payment_id=p."PaymentMethodId" +LEFT JOIN "Invoices" i ON cte.invoice_id=i."Id" +WHERE (p."PaymentMethodId" IS NOT NULL AND p."PaymentMethodId" = payment_method_id) OR + (p."PaymentMethodId" IS NULL AND get_prompt(i."Blob2", payment_method_id) IS NOT NULL AND (get_prompt(i."Blob2", payment_method_id)->'activated')::BOOLEAN IS NOT FALSE); +$$ LANGUAGE SQL STABLE; diff --git a/BTCPayServer.Data/Data/AddressInvoiceData.cs b/BTCPayServer.Data/Data/AddressInvoiceData.cs index 77b8b8684..89fc2d37f 100644 --- a/BTCPayServer.Data/Data/AddressInvoiceData.cs +++ b/BTCPayServer.Data/Data/AddressInvoiceData.cs @@ -19,7 +19,7 @@ namespace BTCPayServer.Data .WithMany(i => i.AddressInvoices).OnDelete(DeleteBehavior.Cascade); builder.Entity() #pragma warning disable CS0618 - .HasKey(o => new { o.PaymentMethodId, o.Address }); + .HasKey(o => new { o.Address, o.PaymentMethodId }); #pragma warning restore CS0618 } } diff --git a/BTCPayServer.Data/Data/PaymentData.Migration.cs b/BTCPayServer.Data/Data/PaymentData.Migration.cs index 3c0397a18..8f9d29499 100644 --- a/BTCPayServer.Data/Data/PaymentData.Migration.cs +++ b/BTCPayServer.Data/Data/PaymentData.Migration.cs @@ -44,9 +44,9 @@ namespace BTCPayServer.Data } var cryptoCode = blob["cryptoCode"].Value(); - Type = cryptoCode + "_" + blob["cryptoPaymentDataType"].Value(); - Type = MigrationExtensions.MigratePaymentMethodId(Type); - var divisibility = MigrationExtensions.GetDivisibility(Type); + PaymentMethodId = cryptoCode + "_" + blob["cryptoPaymentDataType"].Value(); + PaymentMethodId = MigrationExtensions.MigratePaymentMethodId(PaymentMethodId); + var divisibility = MigrationExtensions.GetDivisibility(PaymentMethodId); Currency = blob["cryptoCode"].Value(); blob.Remove("cryptoCode"); blob.Remove("cryptoPaymentDataType"); diff --git a/BTCPayServer.Data/Data/PaymentData.cs b/BTCPayServer.Data/Data/PaymentData.cs index 74ab13ab5..76f2c52b1 100644 --- a/BTCPayServer.Data/Data/PaymentData.cs +++ b/BTCPayServer.Data/Data/PaymentData.cs @@ -27,13 +27,15 @@ namespace BTCPayServer.Data [Obsolete("Use Blob2 instead")] public byte[] Blob { get; set; } public string Blob2 { get; set; } - public string Type { get; set; } + public string PaymentMethodId { get; set; } [Obsolete("Use Status instead")] public bool? Accounted { get; set; } public PaymentStatus? Status { get; set; } public static bool IsPending(PaymentStatus? status) => throw new NotSupportedException(); - internal static void OnModelCreating(ModelBuilder builder, DatabaseFacade databaseFacade) + internal static void OnModelCreating(ModelBuilder builder) { + builder.Entity() + .HasKey(o => new { o.Id, o.PaymentMethodId }); builder.Entity() .HasOne(o => o.InvoiceData) .WithMany(i => i.Payments).OnDelete(DeleteBehavior.Cascade); diff --git a/BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs b/BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs index b17981228..7b34c9dae 100644 --- a/BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs +++ b/BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs @@ -33,13 +33,13 @@ namespace BTCPayServer.Migrations WHEN STRPOS((string_to_array("Address", '#'))[2], '_MoneroLike') > 0 THEN replace((string_to_array("Address", '#'))[2],'_MoneroLike','-CHAIN') WHEN STRPOS((string_to_array("Address", '#'))[2], '_ZcashLike') > 0 THEN replace((string_to_array("Address", '#'))[2],'_ZcashLike','-CHAIN') ELSE '' END; - + ALTER TABLE "AddressInvoices" DROP COLUMN IF EXISTS "CreatedTime"; DELETE FROM "AddressInvoices" WHERE "PaymentMethodId" = ''; """); migrationBuilder.AddPrimaryKey( name: "PK_AddressInvoices", table: "AddressInvoices", - columns: new[] { "PaymentMethodId", "Address" }); + columns: new[] { "Address", "PaymentMethodId" }); migrationBuilder.Sql("VACUUM (ANALYZE) \"AddressInvoices\";", true); } diff --git a/BTCPayServer.Data/Migrations/20240923065254_refactorpayments.cs b/BTCPayServer.Data/Migrations/20240923065254_refactorpayments.cs new file mode 100644 index 000000000..80ad50b00 --- /dev/null +++ b/BTCPayServer.Data/Migrations/20240923065254_refactorpayments.cs @@ -0,0 +1,39 @@ +using BTCPayServer.Data; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace BTCPayServer.Migrations +{ + + [DbContext(typeof(ApplicationDbContext))] + [Migration("20240923065254_refactorpayments")] + [DBScript("005.PaymentsRenaming.sql")] + public partial class refactorpayments : DBScriptsMigration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropPrimaryKey( + name: "PK_Payments", + table: "Payments"); + + migrationBuilder.RenameColumn( + name: "Type", + table: "Payments", + newName: "PaymentMethodId"); + migrationBuilder.Sql("UPDATE \"Payments\" SET \"PaymentMethodId\"='' WHERE \"PaymentMethodId\" IS NULL;"); + migrationBuilder.AddPrimaryKey( + name: "PK_Payments", + table: "Payments", + columns: new[] { "Id", "PaymentMethodId" }); + base.Up(migrationBuilder); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + } + } +} diff --git a/BTCPayServer.Data/Migrations/20240405052858_cleanup_address_invoices.cs b/BTCPayServer.Data/Migrations/20240923071444_temprefactor2.cs similarity index 51% rename from BTCPayServer.Data/Migrations/20240405052858_cleanup_address_invoices.cs rename to BTCPayServer.Data/Migrations/20240923071444_temprefactor2.cs index b5ac18383..6b3406622 100644 --- a/BTCPayServer.Data/Migrations/20240405052858_cleanup_address_invoices.cs +++ b/BTCPayServer.Data/Migrations/20240923071444_temprefactor2.cs @@ -1,6 +1,4 @@ -using System; using BTCPayServer.Data; -using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; @@ -9,17 +7,20 @@ using Microsoft.EntityFrameworkCore.Migrations; namespace BTCPayServer.Migrations { [DbContext(typeof(ApplicationDbContext))] - [Migration("20240405052858_cleanup_address_invoices")] - public partial class cleanup_address_invoices : Migration + [Migration("20240923071444_temprefactor2")] + public partial class temprefactor2 : Migration { /// protected override void Up(MigrationBuilder migrationBuilder) { - migrationBuilder.Sql(@" -DELETE FROM ""AddressInvoices"" WHERE ""Address"" LIKE '%_LightningLike'; -ALTER TABLE ""AddressInvoices"" DROP COLUMN IF EXISTS ""CreatedTime""; -"); - migrationBuilder.Sql(@"VACUUM (FULL, ANALYZE) ""AddressInvoices"";", true); + migrationBuilder.DropPrimaryKey( + name: "PK_AddressInvoices", + table: "AddressInvoices"); + + migrationBuilder.AddPrimaryKey( + name: "PK_AddressInvoices", + table: "AddressInvoices", + columns: new[] { "Address", "PaymentMethodId" }); } /// diff --git a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs index 78bb8d502..c9c22caf6 100644 --- a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs @@ -60,16 +60,16 @@ namespace BTCPayServer.Migrations modelBuilder.Entity("BTCPayServer.Data.AddressInvoiceData", b => { - b.Property("PaymentMethodId") + b.Property("Address") .HasColumnType("text"); - b.Property("Address") + b.Property("PaymentMethodId") .HasColumnType("text"); b.Property("InvoiceDataId") .HasColumnType("text"); - b.HasKey("PaymentMethodId", "Address"); + b.HasKey("Address", "PaymentMethodId"); b.HasIndex("InvoiceDataId"); @@ -482,6 +482,9 @@ namespace BTCPayServer.Migrations b.Property("Id") .HasColumnType("text"); + b.Property("PaymentMethodId") + .HasColumnType("text"); + b.Property("Accounted") .HasColumnType("boolean"); @@ -506,10 +509,7 @@ namespace BTCPayServer.Migrations b.Property("Status") .HasColumnType("text"); - b.Property("Type") - .HasColumnType("text"); - - b.HasKey("Id"); + b.HasKey("Id", "PaymentMethodId"); b.HasIndex("InvoiceDataId"); diff --git a/BTCPayServer.Tests/TestAccount.cs b/BTCPayServer.Tests/TestAccount.cs index 90889ff8e..72168ace8 100644 --- a/BTCPayServer.Tests/TestAccount.cs +++ b/BTCPayServer.Tests/TestAccount.cs @@ -698,11 +698,13 @@ retry: await writer.FlushAsync(); } isHeader = true; - using (var writer = db.BeginTextImport("COPY \"Payments\" (\"Id\",\"Blob\",\"InvoiceDataId\",\"Accounted\",\"Blob2\",\"Type\") FROM STDIN DELIMITER ',' CSV HEADER")) + using (var writer = db.BeginTextImport("COPY \"Payments\" (\"Id\",\"Blob\",\"InvoiceDataId\",\"Accounted\",\"Blob2\",\"PaymentMethodId\") FROM STDIN DELIMITER ',' CSV HEADER")) { foreach (var invoice in oldPayments) { var localPayment = invoice.Replace("3sgUCCtUBg6S8LJkrbdfAWbsJMqByFLfvSqjG6xKBWEd", storeId); + // Old data could have Type to null. + localPayment += "BTC-CHAIN"; await writer.WriteLineAsync(localPayment); } await writer.FlushAsync(); diff --git a/BTCPayServer.Tests/TestData/InvoiceMigrationTestVectors.json b/BTCPayServer.Tests/TestData/InvoiceMigrationTestVectors.json index 0623dbfca..a3dda6d26 100644 --- a/BTCPayServer.Tests/TestData/InvoiceMigrationTestVectors.json +++ b/BTCPayServer.Tests/TestData/InvoiceMigrationTestVectors.json @@ -604,7 +604,7 @@ }, "expectedProperties": { "Created": "04/23/2019 18:27:56 +00:00", - "Type": "BTC-CHAIN", + "PaymentMethodId": "BTC-CHAIN", "Currency": "BTC", "Status": "Settled", "Amount": "0.07299962", @@ -634,7 +634,7 @@ }, "expectedProperties": { "Created": "10/01/2018 14:13:22 +00:00", - "Type": "BTC-CHAIN", + "PaymentMethodId": "BTC-CHAIN", "Currency": "BTC", "Status": "Settled", "Amount": "0.00017863", @@ -666,7 +666,7 @@ "Created": "03/21/2024 07:24:35 +00:00", "CreatedInMs": "1711005875969", "Amount": "0.00000001", - "Type": "BTC-LNURL", + "PaymentMethodId": "BTC-LNURL", "Currency": "BTC" } }, @@ -697,7 +697,7 @@ "Created": "03/20/2024 22:39:08 +00:00", "CreatedInMs": "1710974348741", "Amount": "0.00197864", - "Type": "BTC-CHAIN", + "PaymentMethodId": "BTC-CHAIN", "Currency": "BTC", "Status": "Settled", "Accounted": null diff --git a/BTCPayServer/Controllers/UIInvoiceController.UI.cs b/BTCPayServer/Controllers/UIInvoiceController.UI.cs index 7afeec204..5416cce29 100644 --- a/BTCPayServer/Controllers/UIInvoiceController.UI.cs +++ b/BTCPayServer/Controllers/UIInvoiceController.UI.cs @@ -674,7 +674,7 @@ namespace BTCPayServer.Controllers private async Task> GetAddresses(PaymentMethodId paymentMethodId, string[] selectedItems) { using var ctx = _dbContextFactory.CreateContext(); - return new HashSet(await ctx.AddressInvoices.Where(i => i.PaymentMethodId == paymentMethodId.ToString() && selectedItems.Contains(i.InvoiceDataId)).Select(i => i.Address).ToArrayAsync()); + return new HashSet(await ctx.AddressInvoices.Where(i => selectedItems.Contains(i.InvoiceDataId) && i.PaymentMethodId == paymentMethodId.ToString()).Select(i => i.Address).ToArrayAsync()); } [HttpGet("i/{invoiceId}")] diff --git a/BTCPayServer/Data/PaymentDataExtensions.cs b/BTCPayServer/Data/PaymentDataExtensions.cs index de5302cae..cf001cff5 100644 --- a/BTCPayServer/Data/PaymentDataExtensions.cs +++ b/BTCPayServer/Data/PaymentDataExtensions.cs @@ -34,13 +34,13 @@ namespace BTCPayServer.Data } public static PaymentData SetBlob(this PaymentData paymentData, PaymentMethodId paymentMethodId, PaymentBlob blob) { - paymentData.Type = paymentMethodId.ToString(); + paymentData.PaymentMethodId = paymentMethodId.ToString(); paymentData.Blob2 = JToken.FromObject(blob, InvoiceDataExtensions.DefaultSerializer).ToString(Newtonsoft.Json.Formatting.None); return paymentData; } public static PaymentMethodId GetPaymentMethodId(this PaymentData paymentData) { - return PaymentMethodId.Parse(paymentData.Type); + return PaymentMethodId.Parse(paymentData.PaymentMethodId); } public static PaymentEntity GetBlob(this PaymentData paymentData) { diff --git a/BTCPayServer/Services/Invoices/InvoiceRepository.cs b/BTCPayServer/Services/Invoices/InvoiceRepository.cs index b74a8ffcd..2088bd977 100644 --- a/BTCPayServer/Services/Invoices/InvoiceRepository.cs +++ b/BTCPayServer/Services/Invoices/InvoiceRepository.cs @@ -72,7 +72,7 @@ namespace BTCPayServer.Services.Invoices using var db = _applicationDbContextFactory.CreateContext(); var row = (await db.AddressInvoices .Include(a => a.InvoiceData.Payments) - .Where(a => a.PaymentMethodId == paymentMethodId.ToString() && a.Address == address) + .Where(a => a.Address == address && a.PaymentMethodId == paymentMethodId.ToString()) .Select(a => a.InvoiceData) .FirstOrDefaultAsync()); return row is null ? null : ToEntity(row); @@ -101,7 +101,7 @@ namespace BTCPayServer.Services.Invoices COALESCE(array_agg(to_jsonb(p)) FILTER (WHERE p."Id" IS NOT NULL), '{}') as payments, (array_agg(to_jsonb(i)))[1] as invoice FROM get_monitored_invoices(@pmi) m - LEFT JOIN "Payments" p ON p."Id" = m.payment_id + LEFT JOIN "Payments" p ON p."Id" = m.payment_id AND p."PaymentMethodId" = m.payment_method_id LEFT JOIN "Invoices" i ON i."Id" = m.invoice_id LEFT JOIN "AddressInvoices" ai ON i."Id" = ai."InvoiceDataId" WHERE ai."PaymentMethodId" = @pmi diff --git a/BTCPayServer/Services/Invoices/PaymentService.cs b/BTCPayServer/Services/Invoices/PaymentService.cs index 1a04515c2..3d591cd1d 100644 --- a/BTCPayServer/Services/Invoices/PaymentService.cs +++ b/BTCPayServer/Services/Invoices/PaymentService.cs @@ -49,7 +49,7 @@ namespace BTCPayServer.Services.Invoices if (invoice == null) return null; invoiceEntity = invoice.GetBlob(); - var pmi = PaymentMethodId.Parse(paymentData.Type); + var pmi = PaymentMethodId.Parse(paymentData.PaymentMethodId); PaymentPrompt paymentMethod = invoiceEntity.GetPaymentPrompt(pmi); if (paymentMethod is null || !_handlers.TryGetValue(pmi, out var handler)) return null;