Cleanup data from the InvoiceEvents table (#5904)

BTCPayServer.Data/ApplicationDbContext.cs

@@ -28,7 +28,6 @@ namespace BTCPayServer.Data
         public DbSet<APIKeyData> ApiKeys { get; set; }
         public DbSet<AppData> Apps { get; set; }
         public DbSet<StoredFile> Files { get; set; }
-        public DbSet<InvoiceEventData> InvoiceEvents { get; set; }
         public DbSet<InvoiceSearchData> InvoiceSearches { get; set; }
         public DbSet<InvoiceWebhookDeliveryData> InvoiceWebhookDeliveries { get; set; }
         public DbSet<InvoiceData> Invoices { get; set; }
@@ -75,7 +74,6 @@ namespace BTCPayServer.Data
             APIKeyData.OnModelCreating(builder, Database);
             AppData.OnModelCreating(builder, Database);
             //StoredFile.OnModelCreating(builder);
-            InvoiceEventData.OnModelCreating(builder);
             InvoiceSearchData.OnModelCreating(builder);
             InvoiceWebhookDeliveryData.OnModelCreating(builder);
             InvoiceData.OnModelCreating(builder, Database);

BTCPayServer.Data/Data/InvoiceData.cs

@@ -16,7 +16,6 @@ namespace BTCPayServer.Data
 
         public DateTimeOffset Created { get; set; }
         public List<PaymentData> Payments { get; set; }
-        public List<InvoiceEventData> Events { get; set; }
 
         [Obsolete("Use Blob2 instead")]
         public byte[] Blob { get; set; }

BTCPayServer.Data/Data/InvoiceEventData.cs

@@ -6,28 +6,10 @@ namespace BTCPayServer.Data
     public class InvoiceEventData
     {
         public string InvoiceDataId { get; set; }
-        public InvoiceData InvoiceData { get; set; }
-        public string UniqueId { get; set; }
         public DateTimeOffset Timestamp { get; set; }
         public string Message { get; set; }
         public EventSeverity Severity { get; set; } = EventSeverity.Info;
 
-
-        internal static void OnModelCreating(ModelBuilder builder)
-        {
-            builder.Entity<InvoiceEventData>()
-                   .HasOne(o => o.InvoiceData)
-                   .WithMany(i => i.Events).OnDelete(DeleteBehavior.Cascade);
-            builder.Entity<InvoiceEventData>()
-                .HasKey(o => new
-                {
-                    o.InvoiceDataId,
-#pragma warning disable CS0618
-                    o.UniqueId
-#pragma warning restore CS0618
-                });
-        }
-
         public enum EventSeverity
         {
             Info,

BTCPayServer.Data/Migrations/20240405004015_cleanup_invoice_events.cs

@@ -0,0 +1,31 @@
+using System;
+using BTCPayServer.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace BTCPayServer.Migrations
+{
+    [DbContext(typeof(ApplicationDbContext))]
+    [Migration("20240405004015_cleanup_invoice_events")]
+    public partial class cleanup_invoice_events : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.Sql(@"
+                ALTER TABLE ""InvoiceEvents"" DROP CONSTRAINT IF EXISTS ""PK_InvoiceEvents"";
+                ALTER TABLE ""InvoiceEvents"" DROP COLUMN IF EXISTS ""UniqueId"";
+                CREATE INDEX IF NOT EXISTS ""IX_InvoiceEvents_InvoiceDataId"" ON ""InvoiceEvents""(""InvoiceDataId"");
+                VACUUM (FULL, ANALYZE) ""InvoiceEvents"";
+            ", true);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+        }
+    }
+}

BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs

@@ -302,28 +302,6 @@ namespace BTCPayServer.Migrations
                     b.ToTable("Invoices");
                 });
 
-            modelBuilder.Entity("BTCPayServer.Data.InvoiceEventData", b =>
-                {
-                    b.Property<string>("InvoiceDataId")
-                        .HasColumnType("text");
-
-                    b.Property<string>("UniqueId")
-                        .HasColumnType("text");
-
-                    b.Property<string>("Message")
-                        .HasColumnType("text");
-
-                    b.Property<int>("Severity")
-                        .HasColumnType("integer");
-
-                    b.Property<DateTimeOffset>("Timestamp")
-                        .HasColumnType("timestamp with time zone");
-
-                    b.HasKey("InvoiceDataId", "UniqueId");
-
-                    b.ToTable("InvoiceEvents");
-                });
-
             modelBuilder.Entity("BTCPayServer.Data.InvoiceSearchData", b =>
                 {
                     b.Property<int>("Id")
@@ -1241,17 +1219,6 @@ namespace BTCPayServer.Migrations
                     b.Navigation("StoreData");
                 });
 
-            modelBuilder.Entity("BTCPayServer.Data.InvoiceEventData", b =>
-                {
-                    b.HasOne("BTCPayServer.Data.InvoiceData", "InvoiceData")
-                        .WithMany("Events")
-                        .HasForeignKey("InvoiceDataId")
-                        .OnDelete(DeleteBehavior.Cascade)
-                        .IsRequired();
-
-                    b.Navigation("InvoiceData");
-                });
-
             modelBuilder.Entity("BTCPayServer.Data.InvoiceSearchData", b =>
                 {
                     b.HasOne("BTCPayServer.Data.InvoiceData", "InvoiceData")
@@ -1597,8 +1564,6 @@ namespace BTCPayServer.Migrations
                 {
                     b.Navigation("AddressInvoices");
 
-                    b.Navigation("Events");
-
                     b.Navigation("InvoiceSearchData");
 
                     b.Navigation("Payments");

BTCPayServer/Controllers/UIInvoiceController.UI.cs

@@ -103,7 +103,6 @@ namespace BTCPayServer.Controllers
                 InvoiceId = new[] { invoiceId },
                 UserId = GetUserId(),
                 IncludeAddresses = true,
-                IncludeEvents = true,
                 IncludeArchived = true,
                 IncludeRefunds = true,
             })).FirstOrDefault();
@@ -144,7 +143,7 @@ namespace BTCPayServer.Controllers
                 RedirectUrl = invoice.RedirectURL?.AbsoluteUri,
                 TypedMetadata = invoice.Metadata,
                 StatusException = invoice.ExceptionStatus,
-                Events = invoice.Events,
+                Events = await _InvoiceRepository.GetInvoiceLogs(invoice.Id),
                 Metadata = metaData,
                 Archived = invoice.Archived,
                 HasRefund = invoice.Refunds.Any(),
@@ -610,7 +609,6 @@ namespace BTCPayServer.Controllers
                 InvoiceId = new[] { invoiceId },
                 UserId = GetUserId(),
                 IncludeAddresses = false,
-                IncludeEvents = false,
                 IncludeArchived = true,
             })).FirstOrDefault();
             if (invoice == null)

BTCPayServer/Data/InvoiceDataExtensions.cs

@@ -50,10 +50,6 @@ namespace BTCPayServer.Data
             {
                 entity.AvailableAddressHashes = invoiceData.AddressInvoices.Select(a => a.GetAddress() + a.GetPaymentMethodId()).ToHashSet();
             }
-            if (invoiceData.Events != null)
-            {
-                entity.Events = invoiceData.Events.OrderBy(c => c.Timestamp).ToList();
-            }
             if (invoiceData.Refunds != null)
             {
                 entity.Refunds = invoiceData.Refunds.OrderBy(c => c.PullPaymentData.StartDate).ToList();

BTCPayServer/Models/InvoicingModels/InvoiceDetailsModel.cs

@@ -120,7 +120,7 @@ namespace BTCPayServer.Models.InvoicingModels
         }
         public InvoiceMetadata TypedMetadata { get; set; }
         public DateTimeOffset MonitoringDate { get; internal set; }
-        public List<InvoiceEventData> Events { get; internal set; }
+        public InvoiceEventData[] Events { get; internal set; }
         public string NotificationEmail { get; internal set; }
         public Dictionary<string, object> Metadata { get; set; }
         public Dictionary<string, object> ReceiptData { get; set; }

BTCPayServer/Services/Invoices/InvoiceEntity.cs

@@ -501,8 +501,6 @@ namespace BTCPayServer.Services.Invoices
         public HashSet<string> AvailableAddressHashes { get; set; }
         [JsonProperty]
         public bool ExtendedNotifications { get; set; }
-        [JsonIgnore]
-        public List<InvoiceEventData> Events { get; internal set; }
 
         [JsonProperty]
         public double PaymentTolerance { get; set; }

BTCPayServer/Services/Invoices/InvoiceRepository.cs

@@ -16,6 +16,7 @@ using Microsoft.EntityFrameworkCore;
 using NBitcoin;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
+using Npgsql;
 using Encoders = NBitcoin.DataEncoders.Encoders;
 using InvoiceData = BTCPayServer.Data.InvoiceData;
 
@@ -140,7 +141,7 @@ namespace BTCPayServer.Services.Invoices
 
         public async Task UpdateInvoice(string invoiceId, UpdateCustomerModel data)
         {
-            retry:
+retry:
             using (var ctx = _applicationDbContextFactory.CreateContext())
             {
                 var invoiceData = await ctx.Invoices.FindAsync(invoiceId);
@@ -169,7 +170,7 @@ namespace BTCPayServer.Services.Invoices
 
         public async Task UpdateInvoiceExpiry(string invoiceId, TimeSpan seconds)
         {
-            retry:
+retry:
             await using (var ctx = _applicationDbContextFactory.CreateContext())
             {
                 var invoiceData = await ctx.Invoices.FindAsync(invoiceId);
@@ -199,7 +200,7 @@ namespace BTCPayServer.Services.Invoices
 
         public async Task ExtendInvoiceMonitor(string invoiceId)
         {
-            retry:
+retry:
             using (var ctx = _applicationDbContextFactory.CreateContext())
             {
                 var invoiceData = await ctx.Invoices.FindAsync(invoiceId);
@@ -275,18 +276,23 @@ namespace BTCPayServer.Services.Invoices
         public async Task AddInvoiceLogs(string invoiceId, InvoiceLogs logs)
         {
             await using var context = _applicationDbContextFactory.CreateContext();
-            foreach (var log in logs.ToList())
+            var db = context.Database.GetDbConnection();
-            {
+            var data = logs.ToList().Select(log => new InvoiceEventData()
-                await context.InvoiceEvents.AddAsync(new InvoiceEventData()
             {
                 Severity = log.Severity,
                 InvoiceDataId = invoiceId,
                 Message = log.Log,
-                    Timestamp = log.Timestamp,
+                Timestamp = log.Timestamp
-                    UniqueId = Encoders.Hex.EncodeData(RandomUtils.GetBytes(10))
+            }).ToArray();
-                });
+
+            await db.ExecuteAsync(InsertInvoiceEvent, data);
         }
-            await context.SaveChangesAsync().ConfigureAwait(false);
+
+        public async Task<InvoiceEventData[]> GetInvoiceLogs(string invoiceId)
+        {
+            await using var context = _applicationDbContextFactory.CreateContext();
+            var db = context.Database.GetDbConnection();
+            return (await db.QueryAsync<InvoiceEventData>("SELECT * FROM \"InvoiceEvents\" WHERE \"InvoiceDataId\"=@InvoiceDataId ORDER BY \"Timestamp\"", new { InvoiceDataId = invoiceId })).ToArray();
         }
 
         public Task UpdatePaymentDetails(string invoiceId, IPaymentMethodHandler handler, object details)
@@ -296,7 +302,7 @@ namespace BTCPayServer.Services.Invoices
         }
         public async Task UpdatePaymentDetails(string invoiceId, PaymentMethodId paymentMethodId, JToken details)
         {
-            retry:
+retry:
             using (var context = _applicationDbContextFactory.CreateContext())
             {
                 try
@@ -346,7 +352,7 @@ retry:
         public async Task NewPaymentPrompt(string invoiceId, PaymentMethodContext paymentPromptContext)
         {
             var prompt = paymentPromptContext.Prompt;
-            retry:
+retry:
             using (var context = _applicationDbContextFactory.CreateContext())
             {
                 var invoice = await context.Invoices.FindAsync(invoiceId);
@@ -397,22 +403,27 @@ retry:
             }
         }
 
+        const string InsertInvoiceEvent = "INSERT INTO \"InvoiceEvents\" (\"InvoiceDataId\", \"Severity\", \"Message\", \"Timestamp\") VALUES (@InvoiceDataId, @Severity, @Message, @Timestamp)";
+
         public async Task AddInvoiceEvent(string invoiceId, object evt, InvoiceEventData.EventSeverity severity)
         {
             await using var context = _applicationDbContextFactory.CreateContext();
-            await context.InvoiceEvents.AddAsync(new InvoiceEventData()
+            var conn = context.Database.GetDbConnection();
+            try
+            {
+                await conn.ExecuteAsync(InsertInvoiceEvent,
+                    new InvoiceEventData()
                     {
                         Severity = severity,
                         InvoiceDataId = invoiceId,
                         Message = evt.ToString(),
-                Timestamp = DateTimeOffset.UtcNow,
+                        Timestamp = DateTimeOffset.UtcNow
-                UniqueId = Encoders.Hex.EncodeData(RandomUtils.GetBytes(10))
                     });
-            try
-            {
-                await context.SaveChangesAsync();
             }
-            catch (DbUpdateException) { } // Probably the invoice does not exists anymore
+            catch (Npgsql.NpgsqlException ex) when (ex.SqlState == PostgresErrorCodes.ForeignKeyViolation)
+            {
+                // Invoice does not exists
+            }
         }
 
         public static void AddToTextSearch(ApplicationDbContext context, InvoiceData invoice, params string[] terms)
@@ -446,7 +457,7 @@ retry:
         }
         internal async Task UpdateInvoicePrice(string invoiceId, decimal price)
         {
-            retry:
+retry:
             using (var context = _applicationDbContextFactory.CreateContext())
             {
                 var invoiceData = await context.FindAsync<Data.InvoiceData>(invoiceId).ConfigureAwait(false);
@@ -756,8 +767,6 @@ retry:
             query = query.Include(o => o.Payments);
             if (queryObject.IncludeAddresses)
                 query = query.Include(o => o.AddressInvoices);
-            if (queryObject.IncludeEvents)
-                query = query.Include(o => o.Events);
             if (queryObject.IncludeRefunds)
                 query = query.Include(o => o.Refunds).ThenInclude(refundData => refundData.PullPaymentData);
             var data = await query.AsNoTracking().ToArrayAsync(cancellationToken).ConfigureAwait(false);
@@ -961,8 +970,6 @@ retry:
             set;
         }
         public bool IncludeAddresses { get; set; }
-
-        public bool IncludeEvents { get; set; }
         public bool IncludeArchived { get; set; } = true;
         public bool IncludeRefunds { get; set; }
         public bool OrderByDesc { get; set; } = true;

BTCPayServer/Services/Reporting/ProductsReportProvider.cs

@@ -33,7 +33,6 @@ public class ProductsReportProvider : ReportProvider
         {
             IncludeArchived = true,
             IncludeAddresses = false,
-            IncludeEvents = false,
             IncludeRefunds = false,
             StartDate = queryContext.From,
             EndDate = queryContext.To,

BTCPayServer/Views/UIInvoice/Invoice.cshtml

@@ -579,7 +579,7 @@
                 </section>
             }
 
-            @if (Model.Events is { Count: > 0 })
+            @if (Model.Events is { Length: > 0 })
             {
                 <section class="mt-5 d-print-none">
                     <h3 class="mb-0">Events</h3>