Make Payouts and PullPayments columns JSONB (#5800)

BTCPayServer.Data/ApplicationDbContext.cs

@@ -107,10 +107,10 @@ namespace BTCPayServer.Data
             //PayjoinLock.OnModelCreating(builder);
             PaymentRequestData.OnModelCreating(builder, Database);
             PaymentData.OnModelCreating(builder, Database);
-            PayoutData.OnModelCreating(builder);
+            PayoutData.OnModelCreating(builder, Database);
             PendingInvoiceData.OnModelCreating(builder);
             //PlannedTransaction.OnModelCreating(builder);
-            PullPaymentData.OnModelCreating(builder);
+            PullPaymentData.OnModelCreating(builder, Database);
             RefundData.OnModelCreating(builder);
             SettingData.OnModelCreating(builder, Database);
             StoreSettingData.OnModelCreating(builder, Database);

BTCPayServer.Data/Data/PayoutData.cs

@@ -1,8 +1,11 @@
 using System;
 using System.ComponentModel.DataAnnotations;
 using System.ComponentModel.DataAnnotations.Schema;
+using System.Text;
 using BTCPayServer.Client.Models;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
 using NBitcoin;
 
 namespace BTCPayServer.Data
@@ -21,14 +24,14 @@ namespace BTCPayServer.Data
         [MaxLength(20)]
         [Required]
         public string PaymentMethodId { get; set; }
-        public byte[] Blob { get; set; }
+        public string Blob { get; set; }
-        public byte[] Proof { get; set; }
+        public string Proof { get; set; }
 #nullable enable
         public string? Destination { get; set; }
 #nullable restore
         public StoreData StoreData { get; set; }
 
-        internal static void OnModelCreating(ModelBuilder builder)
+        internal static void OnModelCreating(ModelBuilder builder, DatabaseFacade databaseFacade)
         {
             builder.Entity<PayoutData>()
                 .HasOne(o => o.PullPaymentData)
@@ -43,6 +46,33 @@ namespace BTCPayServer.Data
                 .HasIndex(o => o.State);
             builder.Entity<PayoutData>()
                 .HasIndex(x => new { DestinationId = x.Destination, x.State });
+
+            if (databaseFacade.IsNpgsql())
+            {
+                builder.Entity<PayoutData>()
+                    .Property(o => o.Blob)
+                    .HasColumnType("JSONB");
+                builder.Entity<PayoutData>()
+                    .Property(o => o.Proof)
+                    .HasColumnType("JSONB");
+            }
+            else if (databaseFacade.IsMySql())
+            {
+                builder.Entity<PayoutData>()
+                    .Property(o => o.Blob)
+                    .HasConversion(new ValueConverter<string, byte[]>
+                    (
+                        convertToProviderExpression: (str) => Encoding.UTF8.GetBytes(str),
+                        convertFromProviderExpression: (bytes) => Encoding.UTF8.GetString(bytes)
+                    ));
+                builder.Entity<PayoutData>()
+                    .Property(o => o.Proof)
+                    .HasConversion(new ValueConverter<string, byte[]>
+                    (
+                        convertToProviderExpression: (str) => Encoding.UTF8.GetBytes(str),
+                        convertFromProviderExpression: (bytes) => Encoding.UTF8.GetString(bytes)
+                    ));
+            }
         }
 
         // utility methods

BTCPayServer.Data/Data/PullPaymentData.cs

@@ -3,8 +3,11 @@ using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
 using System.ComponentModel.DataAnnotations.Schema;
 using System.Linq;
+using System.Text;
 using BTCPayServer.Client.Models;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
 using NBitcoin;
 
 namespace BTCPayServer.Data
@@ -24,16 +27,33 @@ namespace BTCPayServer.Data
         public DateTimeOffset? EndDate { get; set; }
         public bool Archived { get; set; }
         public List<PayoutData> Payouts { get; set; }
-        public byte[] Blob { get; set; }
+        public string Blob { get; set; }
 
 
-        internal static void OnModelCreating(ModelBuilder builder)
+        internal static void OnModelCreating(ModelBuilder builder, DatabaseFacade databaseFacade)
         {
             builder.Entity<PullPaymentData>()
                 .HasIndex(o => o.StoreId);
             builder.Entity<PullPaymentData>()
-                .HasOne(o => o.StoreData)
+            .HasOne(o => o.StoreData)
                 .WithMany(o => o.PullPayments).OnDelete(DeleteBehavior.Cascade);
+
+            if (databaseFacade.IsNpgsql())
+            {
+                builder.Entity<PullPaymentData>()
+                    .Property(o => o.Blob)
+                    .HasColumnType("JSONB");
+            }
+            else if (databaseFacade.IsMySql())
+            {
+                builder.Entity<PullPaymentData>()
+                    .Property(o => o.Blob)
+                    .HasConversion(new ValueConverter<string, byte[]>
+                    (
+                        convertToProviderExpression: (str) => Encoding.UTF8.GetBytes(str),
+                        convertFromProviderExpression: (bytes) => Encoding.UTF8.GetString(bytes)
+                    ));
+            }
         }
 
         public (DateTimeOffset Start, DateTimeOffset? End)? GetPeriod(DateTimeOffset now)

BTCPayServer.Data/Migrations/20240229000000_PayoutAndPullPaymentToJsonBlob.cs

@@ -0,0 +1,28 @@
+using BTCPayServer.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace BTCPayServer.Migrations
+{
+    [DbContext(typeof(ApplicationDbContext))]
+    [Migration("20240229000000_PayoutAndPullPaymentToJsonBlob")]
+    public partial class PayoutAndPullPaymentToJsonBlob : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            if (migrationBuilder.IsNpgsql())
+            {
+                migrationBuilder.Sql("ALTER TABLE \"Payouts\" ALTER COLUMN \"Blob\" TYPE JSONB USING regexp_replace(convert_from(\"Blob\",'UTF8'), '\\\\u0000', '', 'g')::JSONB");
+                migrationBuilder.Sql("ALTER TABLE \"Payouts\" ALTER COLUMN \"Proof\" TYPE JSONB USING regexp_replace(convert_from(\"Proof\",'UTF8'), '\\\\u0000', '', 'g')::JSONB");
+                migrationBuilder.Sql("ALTER TABLE \"PullPayments\" ALTER COLUMN \"Blob\" TYPE JSONB USING regexp_replace(convert_from(\"Blob\",'UTF8'), '\\\\u0000', '', 'g')::JSONB");
+            }
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+        }
+    }
+}

BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs

@@ -1,4 +1,4 @@
-// <auto-generated />
+// <auto-generated />
 using System;
 using BTCPayServer.Data;
 using Microsoft.EntityFrameworkCore;
@@ -599,7 +599,7 @@ namespace BTCPayServer.Migrations
                         .HasColumnType("TEXT");
 
                     b.Property<byte[]>("Blob")
-                        .HasColumnType("BLOB");
+                        .HasColumnType("TEXT");
 
                     b.Property<DateTimeOffset>("Date")
                         .HasColumnType("TEXT");
@@ -613,7 +613,7 @@ namespace BTCPayServer.Migrations
                         .HasColumnType("TEXT");
 
                     b.Property<byte[]>("Proof")
-                        .HasColumnType("BLOB");
+                        .HasColumnType("TEXT");
 
                     b.Property<string>("PullPaymentDataId")
                         .HasColumnType("TEXT");
@@ -704,7 +704,7 @@ namespace BTCPayServer.Migrations
                         .HasColumnType("INTEGER");
 
                     b.Property<byte[]>("Blob")
-                        .HasColumnType("BLOB");
+                        .HasColumnType("TEXT");
 
                     b.Property<DateTimeOffset?>("EndDate")
                         .HasColumnType("TEXT");

BTCPayServer/Data/Payouts/BitcoinLike/BitcoinLikePayoutHandler.cs

@@ -134,7 +134,7 @@ public class BitcoinLikePayoutHandler : IPayoutHandler
         return raw.ToObject<ManualPayoutProof>();
     }
 
-    public static void ParseProofType(byte[] proof, out JObject obj, out string type)
+    public static void ParseProofType(string proof, out JObject obj, out string type)
     {
         type = null;
         if (proof is null)
@@ -143,7 +143,7 @@ public class BitcoinLikePayoutHandler : IPayoutHandler
             return;
         }
 
-        obj = JObject.Parse(Encoding.UTF8.GetString(proof));
+        obj = JObject.Parse(proof);
         TryParseProofType(obj, out type);
     }
 

BTCPayServer/Data/Payouts/PayoutExtensions.cs

@@ -44,18 +44,18 @@ namespace BTCPayServer.Data
 
         public static PayoutBlob GetBlob(this PayoutData data, BTCPayNetworkJsonSerializerSettings serializers)
         {
-            var result =  JsonConvert.DeserializeObject<PayoutBlob>(Encoding.UTF8.GetString(data.Blob), serializers.GetSerializer(data.GetPaymentMethodId().CryptoCode));
+            var result =  JsonConvert.DeserializeObject<PayoutBlob>(data.Blob, serializers.GetSerializer(data.GetPaymentMethodId().CryptoCode));
             result.Metadata ??= new JObject();
             return result;
         }
         public static void SetBlob(this PayoutData data, PayoutBlob blob, BTCPayNetworkJsonSerializerSettings serializers)
         {
-            data.Blob = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(blob, serializers.GetSerializer(data.GetPaymentMethodId().CryptoCode)));
+            data.Blob = JsonConvert.SerializeObject(blob, serializers.GetSerializer(data.GetPaymentMethodId().CryptoCode)).ToString();
         }
 
         public static JObject GetProofBlobJson(this PayoutData data)
         {
-            return data?.Proof is null ? null : JObject.Parse(Encoding.UTF8.GetString(data.Proof));
+            return data?.Proof is null ? null : JObject.Parse(data.Proof);
         }
         public static void SetProofBlob(this PayoutData data, IPayoutProof blob, JsonSerializerSettings settings)
         {
@@ -76,11 +76,10 @@ namespace BTCPayServer.Data
                 data.Proof = null;
                 return;
             }
-            var bytes = Encoding.UTF8.GetBytes(blob.ToString(Formatting.None));
             // We only update the property if the bytes actually changed, this prevent from hammering the DB too much
-            if (data.Proof is null || bytes.Length != data.Proof.Length || !bytes.SequenceEqual(data.Proof))
+            if (!JToken.DeepEquals(blob, data.Proof is null ? null : JObject.Parse(data.Proof)))
             {
-                data.Proof = bytes;
+                data.Proof = blob.ToString(Formatting.None);
             }
         }
 

BTCPayServer/Data/PullPayments/PullPaymentsExtensions.cs

@@ -10,13 +10,13 @@ namespace BTCPayServer.Data
 
         public static PullPaymentBlob GetBlob(this PullPaymentData data)
         {
-            var result = JsonConvert.DeserializeObject<PullPaymentBlob>(Encoding.UTF8.GetString(data.Blob));
+            var result = JsonConvert.DeserializeObject<PullPaymentBlob>(data.Blob);
             result!.SupportedPaymentMethods = result.SupportedPaymentMethods.Where(id => id is not null).ToArray();
             return result;
         }
         public static void SetBlob(this PullPaymentData data, PullPaymentBlob blob)
         {
-            data.Blob = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(blob));
+            data.Blob = JsonConvert.SerializeObject(blob).ToString();
         }
 
         public static bool IsSupported(this PullPaymentData data, Payments.PaymentMethodId paymentId)