Remove only dependency on Dbriize (TextSearch in new invoice column) (#2029)

* Remove only dependency on Dbriize (TextSearch in new invoice column)

* Switch to table for invoice text search

* Adding missing using after rebase

* Removing database migration in preparation for refresh

* Database Migration: Adding InvoiceSearchData

* Refactoring InvoicesRepository to make AddToTextSearch static and non-async

Operation as async is too expensive for simple filtering and AddRange

* Renaming InvoiceQuery property to Take

More inline with what property does by convention, Take is used in conjuction with Skip

* Refactoring SettingsRepository so update of settings can happen in another context

* Adding DbMigrationsHostedService that performs long running data migrations

* Commenting special placing of MigrationStartupTask

* Simplifying code and leaving comment on expected flow

* Resolving problems after merge

* Database Migration: Refreshing database migration, ensuring no unintended changes on ModelSnapshot

Co-authored-by: rockstardev <rockstardev@users.noreply.github.com>
Co-authored-by: Nicolas Dorier <nicolas.dorier@gmail.com>

BTCPayServer.Data/Data/ApplicationDbContext.cs

@@ -67,6 +67,7 @@ namespace BTCPayServer.Data
         public DbSet<WebhookData> Webhooks { get; set; }
         public DbSet<WebhookDeliveryData> WebhookDeliveries { get; set; }
         public DbSet<InvoiceWebhookDeliveryData> InvoiceWebhookDeliveries { get; set; }
+        public DbSet<InvoiceSearchData> InvoiceSearchDatas { get; set; }
 
         protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
         {
@@ -81,6 +82,7 @@ namespace BTCPayServer.Data
             Data.UserStore.OnModelCreating(builder);
             NotificationData.OnModelCreating(builder);
             InvoiceData.OnModelCreating(builder);
+            InvoiceSearchData.OnModelCreating(builder);
             PaymentData.OnModelCreating(builder);
             Data.UserStore.OnModelCreating(builder);
             APIKeyData.OnModelCreating(builder);

BTCPayServer.Data/Data/InvoiceData.cs

@@ -75,6 +75,7 @@ namespace BTCPayServer.Data
         }
         public bool Archived { get; set; }
         public List<PendingInvoiceData> PendingInvoices { get; set; }
+        public List<InvoiceSearchData> InvoiceSearchData { get; set; }
         public List<RefundData> Refunds { get; set; }
         public string CurrentRefundId { get; set; }
         [ForeignKey("Id,CurrentRefundId")]

BTCPayServer.Data/Data/InvoiceSearchData.cs

@@ -0,0 +1,38 @@
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+namespace BTCPayServer.Data
+{
+    public class InvoiceSearchData
+    {
+        [DatabaseGenerated(DatabaseGeneratedOption.Identity)]
+        [Key]
+        public int Id { get; set; }
+
+        [ForeignKey(nameof(InvoiceData))]
+        public string InvoiceDataId { get; set; }
+        public InvoiceData InvoiceData { get; set; }
+        public string Value { get; set; }
+
+
+        internal static void OnModelCreating(ModelBuilder builder)
+        {
+            builder.Entity<InvoiceSearchData>()
+                .HasOne(o => o.InvoiceData)
+                .WithMany(a => a.InvoiceSearchData)
+                .OnDelete(DeleteBehavior.Cascade);
+
+            builder.Entity<InvoiceSearchData>()
+                .HasIndex(data => data.Value);
+
+            builder.Entity<InvoiceSearchData>()
+                .Property(a => a.Id)
+                .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn)
+                .HasAnnotation("MySql:ValueGeneratedOnAdd", true)
+                .HasAnnotation("Sqlite:Autoincrement", true);
+        }
+    }
+}

BTCPayServer.Data/Migrations/20201227165824_AdddingInvoiceSearchData.cs

@@ -0,0 +1,55 @@
+using BTCPayServer.Data;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+namespace BTCPayServer.Migrations
+{
+    [DbContext(typeof(ApplicationDbContext))]
+    [Migration("20201227165824_AdddingInvoiceSearchData")]
+    public partial class AdddingInvoiceSearchData : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "InvoiceSearchDatas",
+                columns: table => new
+                {
+                    Id = table.Column<int>(nullable: false)
+                        // manually added
+                        .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn)
+                        .Annotation("MySql:ValueGeneratedOnAdd", true)
+                        .Annotation("Sqlite:Autoincrement", true),
+                        // eof manually added
+                    InvoiceDataId = table.Column<string>(nullable: true),
+                    Value = table.Column<string>(nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_InvoiceSearchDatas", x => x.Id);
+                    table.ForeignKey(
+                        name: "FK_InvoiceSearchDatas_Invoices_InvoiceDataId",
+                        column: x => x.InvoiceDataId,
+                        principalTable: "Invoices",
+                        principalColumn: "Id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
+            migrationBuilder.CreateIndex(
+                name: "IX_InvoiceSearchDatas_InvoiceDataId",
+                table: "InvoiceSearchDatas",
+                column: "InvoiceDataId");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_InvoiceSearchDatas_Value",
+                table: "InvoiceSearchDatas",
+                column: "Value");
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "InvoiceSearchDatas");
+        }
+    }
+}

BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs

@@ -4,6 +4,7 @@ using BTCPayServer.Data;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
 
 namespace BTCPayServer.Migrations
 {
@@ -259,6 +260,30 @@ namespace BTCPayServer.Migrations
                     b.ToTable("InvoiceEvents");
                 });
 
+            modelBuilder.Entity("BTCPayServer.Data.InvoiceSearchData", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER")
+                        .HasAnnotation("MySql:ValueGeneratedOnAdd", true)
+                        .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn)
+                        .HasAnnotation("Sqlite:Autoincrement", true);
+
+                    b.Property<string>("InvoiceDataId")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("Value")
+                        .HasColumnType("TEXT");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("InvoiceDataId");
+
+                    b.HasIndex("Value");
+
+                    b.ToTable("InvoiceSearchDatas");
+                });
+
             modelBuilder.Entity("BTCPayServer.Data.InvoiceWebhookDeliveryData", b =>
                 {
                     b.Property<string>("InvoiceId")
@@ -963,6 +988,14 @@ namespace BTCPayServer.Migrations
                         .IsRequired();
                 });
 
+            modelBuilder.Entity("BTCPayServer.Data.InvoiceSearchData", b =>
+                {
+                    b.HasOne("BTCPayServer.Data.InvoiceData", "InvoiceData")
+                        .WithMany("InvoiceSearchData")
+                        .HasForeignKey("InvoiceDataId")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
             modelBuilder.Entity("BTCPayServer.Data.InvoiceWebhookDeliveryData", b =>
                 {
                     b.HasOne("BTCPayServer.Data.WebhookDeliveryData", "Delivery")

BTCPayServer/Controllers/InvoiceController.API.cs

@@ -69,7 +69,7 @@ namespace BTCPayServer.Controllers
 
             var query = new InvoiceQuery()
             {
-                Count = limit,
+                Take = limit,
                 Skip = offset,
                 EndDate = dateEnd,
                 StartDate = dateStart,

BTCPayServer/Controllers/InvoiceController.UI.cs

@@ -705,7 +705,7 @@ namespace BTCPayServer.Controllers
 
             InvoiceQuery invoiceQuery = GetInvoiceQuery(model.SearchTerm, model.TimezoneOffset ?? 0);
             var counting = _InvoiceRepository.GetInvoicesTotal(invoiceQuery);
-            invoiceQuery.Count = model.Count;
+            invoiceQuery.Take = model.Count;
             invoiceQuery.Skip = model.Skip;
             var list = await _InvoiceRepository.GetInvoices(invoiceQuery);
 
@@ -759,7 +759,7 @@ namespace BTCPayServer.Controllers
 
             InvoiceQuery invoiceQuery = GetInvoiceQuery(searchTerm, timezoneOffset);
             invoiceQuery.Skip = 0;
-            invoiceQuery.Count = int.MaxValue;
+            invoiceQuery.Take = int.MaxValue;
             var invoices = await _InvoiceRepository.GetInvoices(invoiceQuery);
             var res = model.Process(invoices, format);
 

BTCPayServer/HostedServices/DbMigrationsHostedService.cs

@@ -0,0 +1,107 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Threading.Tasks;
+using BTCPayServer.Data;
+using BTCPayServer.Services;
+using BTCPayServer.Services.Invoices;
+
+namespace BTCPayServer.HostedServices
+{
+    /// <summary>
+    /// In charge of all long running db migrations that we can't execute on startup in MigrationStartupTask
+    /// </summary>
+    public class DbMigrationsHostedService : BaseAsyncService
+    {
+        private readonly InvoiceRepository _invoiceRepository;
+        private readonly SettingsRepository _settingsRepository;
+        private readonly ApplicationDbContextFactory _dbContextFactory;
+
+        public DbMigrationsHostedService(InvoiceRepository invoiceRepository, SettingsRepository settingsRepository, ApplicationDbContextFactory dbContextFactory)
+        {
+            _invoiceRepository = invoiceRepository;
+            _settingsRepository = settingsRepository;
+            _dbContextFactory = dbContextFactory;
+        }
+        
+
+        internal override Task[] InitializeTasks()
+        {
+            return new Task[] { ProcessMigration() };
+        }
+
+        protected async Task ProcessMigration()
+        {
+            var settings = await _settingsRepository.GetSettingAsync<MigrationSettings>();
+            if (settings.MigratedInvoiceTextSearchPages != int.MaxValue)
+            {
+                await MigratedInvoiceTextSearchToDb(settings.MigratedInvoiceTextSearchPages.Value);
+            }
+
+            // Refresh settings since these operations may run for very long time
+        }
+
+        private async Task MigratedInvoiceTextSearchToDb(int startFromPage)
+        {
+            using var ctx = _dbContextFactory.CreateContext();
+
+            var invoiceQuery = new InvoiceQuery { IncludeArchived = true };
+            var totalCount = await _invoiceRepository.GetInvoicesTotal(invoiceQuery);
+            const int PAGE_SIZE = 1000;
+            var totalPages = Math.Ceiling(totalCount * 1.0m / PAGE_SIZE);
+            for (int i = startFromPage; i < totalPages; i++)
+            {
+                invoiceQuery.Skip = i * PAGE_SIZE;
+                invoiceQuery.Take = PAGE_SIZE;
+                var invoices = await _invoiceRepository.GetInvoices(invoiceQuery);
+
+                foreach (var invoice in invoices)
+                {
+                    var textSearch = new List<string>();
+
+                    // recreating different textSearch.Adds that were previously in DBriize
+                    foreach (var paymentMethod in invoice.GetPaymentMethods())
+                    {
+                        if (paymentMethod.Network != null)
+                        {
+                            var paymentDestination = paymentMethod.GetPaymentMethodDetails().GetPaymentDestination();
+                            textSearch.Add(paymentDestination);
+                            textSearch.Add(paymentMethod.Calculate().TotalDue.ToString());
+                        }
+                    }
+                    // 
+                    textSearch.Add(invoice.Id);
+                    textSearch.Add(invoice.InvoiceTime.ToString(CultureInfo.InvariantCulture));
+                    textSearch.Add(invoice.Price.ToString(CultureInfo.InvariantCulture));
+                    textSearch.Add(invoice.Metadata.OrderId);
+                    textSearch.Add(InvoiceRepository.ToJsonString(invoice.Metadata, null));
+                    textSearch.Add(invoice.StoreId);
+                    textSearch.Add(invoice.Metadata.BuyerEmail);
+                    //
+                    textSearch.Add(invoice.RefundMail);
+                    // TODO: Are there more things to cache? PaymentData?
+
+                    InvoiceRepository.AddToTextSearch(ctx, 
+                        new InvoiceData { Id = invoice.Id, InvoiceSearchData = new List<InvoiceSearchData>() }, 
+                        textSearch.ToArray());
+                }
+
+                var settings = await _settingsRepository.GetSettingAsync<MigrationSettings>();
+                if (i + 1 < totalPages)
+                {
+                    settings.MigratedInvoiceTextSearchPages = i;
+                }
+                else
+                {
+                    // during final pass we set int.MaxValue so migration doesn't run again
+                    settings.MigratedInvoiceTextSearchPages = int.MaxValue;
+                }
+
+                // this call triggers update; we're sure that MigrationSettings is already initialized in db 
+                // because of logic executed in MigrationStartupTask.cs
+                _settingsRepository.UpdateSettingInContext(ctx, settings);
+                await ctx.SaveChangesAsync();
+            }
+        }
+    }
+}

BTCPayServer/Hosting/BTCPayServerServices.cs

@@ -97,16 +97,15 @@ namespace BTCPayServer.Hosting
             services.TryAddSingleton<InvoicePaymentNotification>();
             services.TryAddSingleton<BTCPayServerOptions>(o =>
                 o.GetRequiredService<IOptions<BTCPayServerOptions>>().Value);
+            // Don't move this StartupTask, we depend on it being right here
             services.AddStartupTask<MigrationStartupTask>();
+            // 
             services.AddStartupTask<BlockExplorerLinkStartupTask>();
             services.TryAddSingleton<InvoiceRepository>(o =>
             {
                 var datadirs = o.GetRequiredService<DataDirectories>();
                 var dbContext = o.GetRequiredService<ApplicationDbContextFactory>();
-                var dbpath = Path.Combine(datadirs.DataDir, "InvoiceDB");
+                return new InvoiceRepository(dbContext, o.GetRequiredService<BTCPayNetworkProvider>(), o.GetService<EventAggregator>());
-                if (!Directory.Exists(dbpath))
-                    Directory.CreateDirectory(dbpath);
-                return new InvoiceRepository(dbContext, dbpath, o.GetRequiredService<BTCPayNetworkProvider>(), o.GetService<EventAggregator>());
             });
             services.AddSingleton<BTCPayServerEnvironment>();
             services.TryAddSingleton<TokenRepository>();
@@ -264,6 +263,8 @@ namespace BTCPayServer.Hosting
             services.AddSingleton<INotificationHandler, InvoiceEventNotification.Handler>();
             services.AddSingleton<INotificationHandler, PayoutNotification.Handler>();
 
+            services.AddSingleton<IHostedService, DbMigrationsHostedService>();
+
             services.AddShopify();
 #if DEBUG
             services.AddSingleton<INotificationHandler, JunkNotification.Handler>();

BTCPayServer/Hosting/MigrationStartupTask.cs

@@ -1,16 +1,23 @@
 using System;
+using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using BTCPayServer.Abstractions.Contracts;
 using BTCPayServer.Client.Models;
+using BTCPayServer.Configuration;
 using BTCPayServer.Data;
 using BTCPayServer.Logging;
 using BTCPayServer.Services;
 using BTCPayServer.Services.Stores;
+using DBriize;
+using DBriize.Utils;
 using Microsoft.AspNetCore.Identity;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
+using NBitcoin.DataEncoders;
 
 namespace BTCPayServer.Hosting
 {
@@ -20,18 +27,21 @@ namespace BTCPayServer.Hosting
         private readonly StoreRepository _StoreRepository;
         private readonly BTCPayNetworkProvider _NetworkProvider;
         private readonly SettingsRepository _Settings;
+        private readonly BTCPayServerOptions _btcPayServerOptions;
         private readonly UserManager<ApplicationUser> _userManager;
         public MigrationStartupTask(
             BTCPayNetworkProvider networkProvider,
             StoreRepository storeRepository,
             ApplicationDbContextFactory dbContextFactory,
             UserManager<ApplicationUser> userManager,
-            SettingsRepository settingsRepository)
+            SettingsRepository settingsRepository,
+            BTCPayServerOptions btcPayServerOptions)
         {
             _DBContextFactory = dbContextFactory;
             _StoreRepository = storeRepository;
             _NetworkProvider = networkProvider;
             _Settings = settingsRepository;
+            _btcPayServerOptions = btcPayServerOptions;
             _userManager = userManager;
         }
         public async Task ExecuteAsync(CancellationToken cancellationToken = default)

BTCPayServer/Services/Invoices/InvoiceRepository.cs

@@ -2,16 +2,13 @@ using System;
 using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
-using System.Runtime.CompilerServices;
 using System.Threading.Tasks;
 using BTCPayServer.Client.Models;
 using BTCPayServer.Data;
 using BTCPayServer.Events;
 using BTCPayServer.Logging;
 using BTCPayServer.Models.InvoicingModels;
-using BTCPayServer.Models.StoreViewModels;
 using BTCPayServer.Payments;
-using DBriize;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 using NBitcoin;
@@ -22,7 +19,7 @@ using InvoiceData = BTCPayServer.Data.InvoiceData;
 
 namespace BTCPayServer.Services.Invoices
 {
-    public class InvoiceRepository : IDisposable
+    public class InvoiceRepository
     {
         static JsonSerializerSettings DefaultSerializerSettings;
         static InvoiceRepository()
@@ -31,31 +28,13 @@ namespace BTCPayServer.Services.Invoices
             NBitcoin.JsonConverters.Serializer.RegisterFrontConverters(DefaultSerializerSettings);
         }
 
-        private readonly DBriizeEngine _Engine;
-        public DBriizeEngine Engine
-        {
-            get
-            {
-                return _Engine;
-            }
-        }
-
         private readonly ApplicationDbContextFactory _ContextFactory;
         private readonly EventAggregator _eventAggregator;
         private readonly BTCPayNetworkProvider _Networks;
-        private readonly CustomThreadPool _IndexerThread;
 
-        public InvoiceRepository(ApplicationDbContextFactory contextFactory, string dbreezePath,
+        public InvoiceRepository(ApplicationDbContextFactory contextFactory,
             BTCPayNetworkProvider networks, EventAggregator eventAggregator)
         {
-            int retryCount = 0;
-retry:
-            try
-            {
-                _Engine = new DBriizeEngine(dbreezePath);
-            }
-            catch when (retryCount++ < 5) { goto retry; }
-            _IndexerThread = new CustomThreadPool(1, "Invoice Indexer");
             _ContextFactory = contextFactory;
             _Networks = networks;
             _eventAggregator = eventAggregator;
@@ -148,7 +127,7 @@ retry:
                 if (invoiceData.CustomerEmail == null && data.Email != null)
                 {
                     invoiceData.CustomerEmail = data.Email;
-                    AddToTextSearch(invoiceId, invoiceData.CustomerEmail);
+                    AddToTextSearch(ctx, invoiceData, invoiceData.CustomerEmail);
                 }
                 await ctx.SaveChangesAsync().ConfigureAwait(false);
             }
@@ -170,7 +149,7 @@ retry:
 
         public async Task<InvoiceEntity> CreateInvoiceAsync(string storeId, InvoiceEntity invoice)
         {
-            List<string> textSearch = new List<string>();
+            var textSearch = new List<string>();
             invoice = Clone(invoice);
             invoice.Networks = _Networks;
             invoice.Id = Encoders.Base58.EncodeData(RandomUtils.GetBytes(16));
@@ -180,7 +159,7 @@ retry:
             invoice.StoreId = storeId;
             using (var context = _ContextFactory.CreateContext())
             {
-                context.Invoices.Add(new Data.InvoiceData()
+                var invoiceData = new Data.InvoiceData()
                 {
                     StoreDataId = storeId,
                     Id = invoice.Id,
@@ -193,7 +172,9 @@ retry:
                     ItemCode = invoice.Metadata.ItemCode,
                     CustomerEmail = invoice.RefundMail,
                     Archived = false
-                });
+                };
+                await context.Invoices.AddAsync(invoiceData);
+
 
                 foreach (var paymentMethod in invoice.GetPaymentMethods())
                 {
@@ -202,13 +183,13 @@ retry:
                     var paymentDestination = paymentMethod.GetPaymentMethodDetails().GetPaymentDestination();
 
                     string address = GetDestination(paymentMethod);
-                    context.AddressInvoices.Add(new AddressInvoiceData()
+                    await context.AddressInvoices.AddAsync(new AddressInvoiceData()
                     {
                         InvoiceDataId = invoice.Id,
                         CreatedTime = DateTimeOffset.UtcNow,
                     }.Set(address, paymentMethod.GetId()));
 
-                    context.HistoricalAddressInvoices.Add(new HistoricalAddressInvoiceData()
+                    await context.HistoricalAddressInvoices.AddAsync(new HistoricalAddressInvoiceData()
                     {
                         InvoiceDataId = invoice.Id,
                         Assigned = DateTimeOffset.UtcNow
@@ -216,18 +197,21 @@ retry:
                     textSearch.Add(paymentDestination);
                     textSearch.Add(paymentMethod.Calculate().TotalDue.ToString());
                 }
-                context.PendingInvoices.Add(new PendingInvoiceData() { Id = invoice.Id });
+                await context.PendingInvoices.AddAsync(new PendingInvoiceData() { Id = invoice.Id });
+
+                textSearch.Add(invoice.Id);
+                textSearch.Add(invoice.InvoiceTime.ToString(CultureInfo.InvariantCulture));
+                textSearch.Add(invoice.Price.ToString(CultureInfo.InvariantCulture));
+                textSearch.Add(invoice.Metadata.OrderId);
+                textSearch.Add(ToJsonString(invoice.Metadata, null));
+                textSearch.Add(invoice.StoreId);
+                textSearch.Add(invoice.Metadata.BuyerEmail);
+                AddToTextSearch(context, invoiceData, textSearch.ToArray());
+
                 await context.SaveChangesAsync().ConfigureAwait(false);
             }
 
-            textSearch.Add(invoice.Id);
+
-            textSearch.Add(invoice.InvoiceTime.ToString(CultureInfo.InvariantCulture));
-            textSearch.Add(invoice.Price.ToString(CultureInfo.InvariantCulture));
-            textSearch.Add(invoice.Metadata.OrderId);
-            textSearch.Add(ToString(invoice.Metadata, null));
-            textSearch.Add(invoice.StoreId);
-            textSearch.Add(invoice.Metadata.BuyerEmail);
-            AddToTextSearch(invoice.Id, textSearch.ToArray());
             return invoice;
         }
 
@@ -295,10 +279,10 @@ retry:
             invoice.Blob = ToBytes(invoiceEntity, network);
 
             await context.AddressInvoices.AddAsync(new AddressInvoiceData()
-                {
+            {
-                    InvoiceDataId = invoiceId,
+                InvoiceDataId = invoiceId,
-                    CreatedTime = DateTimeOffset.UtcNow
+                CreatedTime = DateTimeOffset.UtcNow
-                }
+            }
                 .Set(GetDestination(paymentMethod), paymentMethod.GetId()));
             await context.HistoricalAddressInvoices.AddAsync(new HistoricalAddressInvoiceData()
             {
@@ -306,8 +290,8 @@ retry:
                 Assigned = DateTimeOffset.UtcNow
             }.SetAddress(paymentMethodDetails.GetPaymentDestination(), network.CryptoCode));
 
+            AddToTextSearch(context, invoice, paymentMethodDetails.GetPaymentDestination());
             await context.SaveChangesAsync();
-            AddToTextSearch(invoice.Id, paymentMethodDetails.GetPaymentDestination());
             return true;
         }
 
@@ -357,7 +341,7 @@ retry:
                 data.UnAssigned == null);
             foreach (var historicalAddressInvoiceData in addresses)
             {
-                historicalAddressInvoiceData.UnAssigned = DateTimeOffset.UtcNow;   
+                historicalAddressInvoiceData.UnAssigned = DateTimeOffset.UtcNow;
             }
         }
 
@@ -372,29 +356,13 @@ retry:
             catch (DbUpdateException) { } //Possibly, it was unassigned before
         }
 
-        private string[] SearchInvoice(string searchTerms)
+        public static void AddToTextSearch(ApplicationDbContext context, InvoiceData invoice, params string[] terms)
         {
-            using (var tx = _Engine.GetTransaction())
+            var filteredTerms = terms.Where(t => !string.IsNullOrWhiteSpace(t)
-            {
+                && (invoice.InvoiceSearchData == null || invoice.InvoiceSearchData.All(data => data.Value != t)))
-                var terms = searchTerms.Split(null);
+                .Distinct()
-                searchTerms = string.Join(' ', terms.Select(t => t.Length > 50 ? t.Substring(0, 50) : t).ToArray());
+                .Select(s => new InvoiceSearchData() { InvoiceDataId = invoice.Id, Value = s });
-                return tx.TextSearch("InvoiceSearch").Block(searchTerms)
+            context.AddRange(filteredTerms);
-                    .GetDocumentIDs()
-                    .Select(id => Encoders.Base58.EncodeData(id))
-                    .ToArray();
-            }
-        }
-
-        void AddToTextSearch(string invoiceId, params string[] terms)
-        {
-            _IndexerThread.DoAsync(() =>
-            {
-                using (var tx = _Engine.GetTransaction())
-                {
-                    tx.TextAppend("InvoiceSearch", Encoders.Base58.DecodeData(invoiceId), string.Join(" ", terms.Where(t => !string.IsNullOrWhiteSpace(t))));
-                    tx.Commit();
-                }
-            });
         }
 
         public async Task UpdateInvoiceStatus(string invoiceId, InvoiceState invoiceState)
@@ -415,7 +383,8 @@ retry:
             using (var context = _ContextFactory.CreateContext())
             {
                 var items = context.Invoices.Where(a => invoiceIds.Contains(a.Id));
-                if (items == null) {
+                if (items == null)
+                {
                     return;
                 }
 
@@ -423,7 +392,7 @@ retry:
                 {
                     invoice.Archived = true;
                 }
-                
+
                 await context.SaveChangesAsync();
             }
         }
@@ -441,7 +410,7 @@ retry:
                 await context.SaveChangesAsync().ConfigureAwait(false);
             }
         }
-        public async Task<InvoiceEntity> UpdateInvoiceMetadata(string invoiceId,  string storeId, JObject metadata)
+        public async Task<InvoiceEntity> UpdateInvoiceMetadata(string invoiceId, string storeId, JObject metadata)
         {
             using (var context = _ContextFactory.CreateContext())
             {
@@ -451,7 +420,7 @@ retry:
                                                 StringComparison.InvariantCultureIgnoreCase)))
                     return null;
                 var blob = invoiceData.GetBlob(_Networks);
-                blob.Metadata =  InvoiceMetadata.FromJObject(metadata);
+                blob.Metadata = InvoiceMetadata.FromJObject(metadata);
                 invoiceData.Blob = ToBytes(blob);
                 await context.SaveChangesAsync().ConfigureAwait(false);
                 return ToEntity(invoiceData);
@@ -595,9 +564,11 @@ retry:
 
         private IQueryable<Data.InvoiceData> GetInvoiceQuery(ApplicationDbContext context, InvoiceQuery queryObject)
         {
-            IQueryable<Data.InvoiceData> query = queryObject.UserId is null 
+            IQueryable<Data.InvoiceData> query = queryObject.UserId is null
                 ? context.Invoices
-                : context.UserStore.Where(u => u.ApplicationUserId == queryObject.UserId).SelectMany(c => c.StoreData.Invoices);
+                : context.UserStore
+                    .Where(u => u.ApplicationUserId == queryObject.UserId)
+                    .SelectMany(c => c.StoreData.Invoices);
 
             if (!queryObject.IncludeArchived)
             {
@@ -618,14 +589,9 @@ retry:
 
             if (!string.IsNullOrEmpty(queryObject.TextSearch))
             {
-                var ids = new HashSet<string>(SearchInvoice(queryObject.TextSearch)).ToArray();
+#pragma warning disable CA1307 // Specify StringComparison
-                if (ids.Length == 0)
+                query = query.Where(i => i.InvoiceSearchData.Any(data => data.Value.StartsWith(queryObject.TextSearch)));
-                {
+#pragma warning restore CA1307 // Specify StringComparison
-                    // Hacky way to return an empty query object. The nice way is much too elaborate:
-                    // https://stackoverflow.com/questions/33305495/how-to-return-empty-iqueryable-in-an-async-repository-method
-                    return query.Where(x => false);
-                }
-                query = query.Where(i => ids.Contains(i.Id));
             }
 
             if (queryObject.StartDate != null)
@@ -668,8 +634,8 @@ retry:
             if (queryObject.Skip != null)
                 query = query.Skip(queryObject.Skip.Value);
 
-            if (queryObject.Count != null)
+            if (queryObject.Take != null)
-                query = query.Take(queryObject.Count.Value);
+                query = query.Take(queryObject.Take.Value);
             return query;
         }
 
@@ -771,12 +737,12 @@ retry:
 
                 await context.Payments.AddAsync(data);
 
+                AddToTextSearch(context, invoice, paymentData.GetSearchTerms());
                 try
                 {
                     await context.SaveChangesAsync().ConfigureAwait(false);
                 }
                 catch (DbUpdateException) { return null; } // Already exists
-                AddToTextSearch(invoiceId, paymentData.GetSearchTerms());
                 return entity;
             }
         }
@@ -802,12 +768,12 @@ retry:
             }
         }
 
-        private byte[] ToBytes<T>(T obj, BTCPayNetworkBase network = null)
+        private static byte[] ToBytes<T>(T obj, BTCPayNetworkBase network = null)
         {
-            return ZipUtils.Zip(ToString(obj, network));
+            return ZipUtils.Zip(ToJsonString(obj, network));
         }
 
-        private string ToString<T>(T data, BTCPayNetworkBase network)
+        public static string ToJsonString<T>(T data, BTCPayNetworkBase network)
         {
             if (network == null)
             {
@@ -815,14 +781,6 @@ retry:
             }
             return network.ToString(data);
         }
-
-        public void Dispose()
-        {
-            if (_Engine != null)
-                _Engine.Dispose();
-            if (_IndexerThread != null)
-                _IndexerThread.Dispose();
-        }
     }
 
     public class InvoiceQuery
@@ -854,7 +812,7 @@ retry:
             get; set;
         }
 
-        public int? Count
+        public int? Take
         {
             get; set;
         }

BTCPayServer/Services/MigrationSettings.cs

@@ -13,5 +13,8 @@ namespace BTCPayServer.Services
         {
             return string.Empty;
         }
+        
+        // Done in DbMigrationsHostedService
+        public int? MigratedInvoiceTextSearchPages { get; set; }
     }
 }

BTCPayServer/Services/SettingsRepository.cs

@@ -30,17 +30,11 @@ namespace BTCPayServer.Services
                 return Deserialize<T>(data.Value);
             }
         }
-
         public async Task UpdateSetting<T>(T obj, string name = null)
         {
-            name ??= obj.GetType().FullName;
             using (var ctx = _ContextFactory.CreateContext())
             {
-                var settings = new SettingData();
+                var settings = UpdateSettingInContext<T>(ctx, obj, name);
-                settings.Id = name;
-                settings.Value = Serialize(obj);
-                ctx.Attach(settings);
-                ctx.Entry(settings).State = EntityState.Modified;
                 try
                 {
                     await ctx.SaveChangesAsync();
@@ -55,7 +49,19 @@ namespace BTCPayServer.Services
             {
                 Settings = obj
             });
+        }
 
+        public SettingData UpdateSettingInContext<T>(ApplicationDbContext ctx, T obj, string name = null)
+        {
+            name ??= obj.GetType().FullName;
+            var settings = new SettingData();
+            settings.Id = name;
+            settings.Value = Serialize(obj);
+
+            ctx.Attach(settings);
+            ctx.Entry(settings).State = EntityState.Modified;
+            
+            return settings;
         }
 
         private T Deserialize<T>(string value)