From 39b54628096ffe65630237853aa243658464760c Mon Sep 17 00:00:00 2001 From: Andrew Camilleri Date: Mon, 28 Dec 2020 11:10:53 +0100 Subject: [PATCH] Remove only dependency on Dbriize (TextSearch in new invoice column) (#2029) * Remove only dependency on Dbriize (TextSearch in new invoice column) * Switch to table for invoice text search * Adding missing using after rebase * Removing database migration in preparation for refresh * Database Migration: Adding InvoiceSearchData * Refactoring InvoicesRepository to make AddToTextSearch static and non-async Operation as async is too expensive for simple filtering and AddRange * Renaming InvoiceQuery property to Take More inline with what property does by convention, Take is used in conjuction with Skip * Refactoring SettingsRepository so update of settings can happen in another context * Adding DbMigrationsHostedService that performs long running data migrations * Commenting special placing of MigrationStartupTask * Simplifying code and leaving comment on expected flow * Resolving problems after merge * Database Migration: Refreshing database migration, ensuring no unintended changes on ModelSnapshot Co-authored-by: rockstardev Co-authored-by: Nicolas Dorier --- .../Data/ApplicationDbContext.cs | 2 + BTCPayServer.Data/Data/InvoiceData.cs | 1 + BTCPayServer.Data/Data/InvoiceSearchData.cs | 38 +++++ ...20201227165824_AdddingInvoiceSearchData.cs | 55 +++++++ .../ApplicationDbContextModelSnapshot.cs | 33 ++++ .../Controllers/InvoiceController.API.cs | 2 +- .../Controllers/InvoiceController.UI.cs | 4 +- .../DbMigrationsHostedService.cs | 107 +++++++++++++ BTCPayServer/Hosting/BTCPayServerServices.cs | 9 +- BTCPayServer/Hosting/MigrationStartupTask.cs | 12 +- .../Services/Invoices/InvoiceRepository.cs | 148 +++++++----------- BTCPayServer/Services/MigrationSettings.cs | 3 + BTCPayServer/Services/SettingsRepository.cs | 20 ++- 13 files changed, 324 insertions(+), 110 deletions(-) create mode 100644 BTCPayServer.Data/Data/InvoiceSearchData.cs create mode 100644 BTCPayServer.Data/Migrations/20201227165824_AdddingInvoiceSearchData.cs create mode 100644 BTCPayServer/HostedServices/DbMigrationsHostedService.cs diff --git a/BTCPayServer.Data/Data/ApplicationDbContext.cs b/BTCPayServer.Data/Data/ApplicationDbContext.cs index fe857548a..33ce962df 100644 --- a/BTCPayServer.Data/Data/ApplicationDbContext.cs +++ b/BTCPayServer.Data/Data/ApplicationDbContext.cs @@ -67,6 +67,7 @@ namespace BTCPayServer.Data public DbSet Webhooks { get; set; } public DbSet WebhookDeliveries { get; set; } public DbSet InvoiceWebhookDeliveries { get; set; } + public DbSet InvoiceSearchDatas { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { @@ -81,6 +82,7 @@ namespace BTCPayServer.Data Data.UserStore.OnModelCreating(builder); NotificationData.OnModelCreating(builder); InvoiceData.OnModelCreating(builder); + InvoiceSearchData.OnModelCreating(builder); PaymentData.OnModelCreating(builder); Data.UserStore.OnModelCreating(builder); APIKeyData.OnModelCreating(builder); diff --git a/BTCPayServer.Data/Data/InvoiceData.cs b/BTCPayServer.Data/Data/InvoiceData.cs index 04a53348f..ecbb64a2d 100644 --- a/BTCPayServer.Data/Data/InvoiceData.cs +++ b/BTCPayServer.Data/Data/InvoiceData.cs @@ -75,6 +75,7 @@ namespace BTCPayServer.Data } public bool Archived { get; set; } public List PendingInvoices { get; set; } + public List InvoiceSearchData { get; set; } public List Refunds { get; set; } public string CurrentRefundId { get; set; } [ForeignKey("Id,CurrentRefundId")] diff --git a/BTCPayServer.Data/Data/InvoiceSearchData.cs b/BTCPayServer.Data/Data/InvoiceSearchData.cs new file mode 100644 index 000000000..464c647ca --- /dev/null +++ b/BTCPayServer.Data/Data/InvoiceSearchData.cs @@ -0,0 +1,38 @@ +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +namespace BTCPayServer.Data +{ + public class InvoiceSearchData + { + [DatabaseGenerated(DatabaseGeneratedOption.Identity)] + [Key] + public int Id { get; set; } + + [ForeignKey(nameof(InvoiceData))] + public string InvoiceDataId { get; set; } + public InvoiceData InvoiceData { get; set; } + public string Value { get; set; } + + + internal static void OnModelCreating(ModelBuilder builder) + { + builder.Entity() + .HasOne(o => o.InvoiceData) + .WithMany(a => a.InvoiceSearchData) + .OnDelete(DeleteBehavior.Cascade); + + builder.Entity() + .HasIndex(data => data.Value); + + builder.Entity() + .Property(a => a.Id) + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn) + .HasAnnotation("MySql:ValueGeneratedOnAdd", true) + .HasAnnotation("Sqlite:Autoincrement", true); + } + } +} diff --git a/BTCPayServer.Data/Migrations/20201227165824_AdddingInvoiceSearchData.cs b/BTCPayServer.Data/Migrations/20201227165824_AdddingInvoiceSearchData.cs new file mode 100644 index 000000000..ea19861ca --- /dev/null +++ b/BTCPayServer.Data/Migrations/20201227165824_AdddingInvoiceSearchData.cs @@ -0,0 +1,55 @@ +using BTCPayServer.Data; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +namespace BTCPayServer.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20201227165824_AdddingInvoiceSearchData")] + public partial class AdddingInvoiceSearchData : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "InvoiceSearchDatas", + columns: table => new + { + Id = table.Column(nullable: false) + // manually added + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn) + .Annotation("MySql:ValueGeneratedOnAdd", true) + .Annotation("Sqlite:Autoincrement", true), + // eof manually added + InvoiceDataId = table.Column(nullable: true), + Value = table.Column(nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_InvoiceSearchDatas", x => x.Id); + table.ForeignKey( + name: "FK_InvoiceSearchDatas_Invoices_InvoiceDataId", + column: x => x.InvoiceDataId, + principalTable: "Invoices", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_InvoiceSearchDatas_InvoiceDataId", + table: "InvoiceSearchDatas", + column: "InvoiceDataId"); + + migrationBuilder.CreateIndex( + name: "IX_InvoiceSearchDatas_Value", + table: "InvoiceSearchDatas", + column: "Value"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "InvoiceSearchDatas"); + } + } +} diff --git a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs index 8041007cf..b439507d0 100644 --- a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs @@ -4,6 +4,7 @@ using BTCPayServer.Data; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; namespace BTCPayServer.Migrations { @@ -259,6 +260,30 @@ namespace BTCPayServer.Migrations b.ToTable("InvoiceEvents"); }); + modelBuilder.Entity("BTCPayServer.Data.InvoiceSearchData", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasAnnotation("MySql:ValueGeneratedOnAdd", true) + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn) + .HasAnnotation("Sqlite:Autoincrement", true); + + b.Property("InvoiceDataId") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("InvoiceDataId"); + + b.HasIndex("Value"); + + b.ToTable("InvoiceSearchDatas"); + }); + modelBuilder.Entity("BTCPayServer.Data.InvoiceWebhookDeliveryData", b => { b.Property("InvoiceId") @@ -963,6 +988,14 @@ namespace BTCPayServer.Migrations .IsRequired(); }); + modelBuilder.Entity("BTCPayServer.Data.InvoiceSearchData", b => + { + b.HasOne("BTCPayServer.Data.InvoiceData", "InvoiceData") + .WithMany("InvoiceSearchData") + .HasForeignKey("InvoiceDataId") + .OnDelete(DeleteBehavior.Cascade); + }); + modelBuilder.Entity("BTCPayServer.Data.InvoiceWebhookDeliveryData", b => { b.HasOne("BTCPayServer.Data.WebhookDeliveryData", "Delivery") diff --git a/BTCPayServer/Controllers/InvoiceController.API.cs b/BTCPayServer/Controllers/InvoiceController.API.cs index 4d6db5029..8d9f06861 100644 --- a/BTCPayServer/Controllers/InvoiceController.API.cs +++ b/BTCPayServer/Controllers/InvoiceController.API.cs @@ -69,7 +69,7 @@ namespace BTCPayServer.Controllers var query = new InvoiceQuery() { - Count = limit, + Take = limit, Skip = offset, EndDate = dateEnd, StartDate = dateStart, diff --git a/BTCPayServer/Controllers/InvoiceController.UI.cs b/BTCPayServer/Controllers/InvoiceController.UI.cs index 72b34d5ff..fd9eb3caf 100644 --- a/BTCPayServer/Controllers/InvoiceController.UI.cs +++ b/BTCPayServer/Controllers/InvoiceController.UI.cs @@ -705,7 +705,7 @@ namespace BTCPayServer.Controllers InvoiceQuery invoiceQuery = GetInvoiceQuery(model.SearchTerm, model.TimezoneOffset ?? 0); var counting = _InvoiceRepository.GetInvoicesTotal(invoiceQuery); - invoiceQuery.Count = model.Count; + invoiceQuery.Take = model.Count; invoiceQuery.Skip = model.Skip; var list = await _InvoiceRepository.GetInvoices(invoiceQuery); @@ -759,7 +759,7 @@ namespace BTCPayServer.Controllers InvoiceQuery invoiceQuery = GetInvoiceQuery(searchTerm, timezoneOffset); invoiceQuery.Skip = 0; - invoiceQuery.Count = int.MaxValue; + invoiceQuery.Take = int.MaxValue; var invoices = await _InvoiceRepository.GetInvoices(invoiceQuery); var res = model.Process(invoices, format); diff --git a/BTCPayServer/HostedServices/DbMigrationsHostedService.cs b/BTCPayServer/HostedServices/DbMigrationsHostedService.cs new file mode 100644 index 000000000..fb114ae96 --- /dev/null +++ b/BTCPayServer/HostedServices/DbMigrationsHostedService.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading.Tasks; +using BTCPayServer.Data; +using BTCPayServer.Services; +using BTCPayServer.Services.Invoices; + +namespace BTCPayServer.HostedServices +{ + /// + /// In charge of all long running db migrations that we can't execute on startup in MigrationStartupTask + /// + public class DbMigrationsHostedService : BaseAsyncService + { + private readonly InvoiceRepository _invoiceRepository; + private readonly SettingsRepository _settingsRepository; + private readonly ApplicationDbContextFactory _dbContextFactory; + + public DbMigrationsHostedService(InvoiceRepository invoiceRepository, SettingsRepository settingsRepository, ApplicationDbContextFactory dbContextFactory) + { + _invoiceRepository = invoiceRepository; + _settingsRepository = settingsRepository; + _dbContextFactory = dbContextFactory; + } + + + internal override Task[] InitializeTasks() + { + return new Task[] { ProcessMigration() }; + } + + protected async Task ProcessMigration() + { + var settings = await _settingsRepository.GetSettingAsync(); + if (settings.MigratedInvoiceTextSearchPages != int.MaxValue) + { + await MigratedInvoiceTextSearchToDb(settings.MigratedInvoiceTextSearchPages.Value); + } + + // Refresh settings since these operations may run for very long time + } + + private async Task MigratedInvoiceTextSearchToDb(int startFromPage) + { + using var ctx = _dbContextFactory.CreateContext(); + + var invoiceQuery = new InvoiceQuery { IncludeArchived = true }; + var totalCount = await _invoiceRepository.GetInvoicesTotal(invoiceQuery); + const int PAGE_SIZE = 1000; + var totalPages = Math.Ceiling(totalCount * 1.0m / PAGE_SIZE); + for (int i = startFromPage; i < totalPages; i++) + { + invoiceQuery.Skip = i * PAGE_SIZE; + invoiceQuery.Take = PAGE_SIZE; + var invoices = await _invoiceRepository.GetInvoices(invoiceQuery); + + foreach (var invoice in invoices) + { + var textSearch = new List(); + + // recreating different textSearch.Adds that were previously in DBriize + foreach (var paymentMethod in invoice.GetPaymentMethods()) + { + if (paymentMethod.Network != null) + { + var paymentDestination = paymentMethod.GetPaymentMethodDetails().GetPaymentDestination(); + textSearch.Add(paymentDestination); + textSearch.Add(paymentMethod.Calculate().TotalDue.ToString()); + } + } + // + textSearch.Add(invoice.Id); + textSearch.Add(invoice.InvoiceTime.ToString(CultureInfo.InvariantCulture)); + textSearch.Add(invoice.Price.ToString(CultureInfo.InvariantCulture)); + textSearch.Add(invoice.Metadata.OrderId); + textSearch.Add(InvoiceRepository.ToJsonString(invoice.Metadata, null)); + textSearch.Add(invoice.StoreId); + textSearch.Add(invoice.Metadata.BuyerEmail); + // + textSearch.Add(invoice.RefundMail); + // TODO: Are there more things to cache? PaymentData? + + InvoiceRepository.AddToTextSearch(ctx, + new InvoiceData { Id = invoice.Id, InvoiceSearchData = new List() }, + textSearch.ToArray()); + } + + var settings = await _settingsRepository.GetSettingAsync(); + if (i + 1 < totalPages) + { + settings.MigratedInvoiceTextSearchPages = i; + } + else + { + // during final pass we set int.MaxValue so migration doesn't run again + settings.MigratedInvoiceTextSearchPages = int.MaxValue; + } + + // this call triggers update; we're sure that MigrationSettings is already initialized in db + // because of logic executed in MigrationStartupTask.cs + _settingsRepository.UpdateSettingInContext(ctx, settings); + await ctx.SaveChangesAsync(); + } + } + } +} diff --git a/BTCPayServer/Hosting/BTCPayServerServices.cs b/BTCPayServer/Hosting/BTCPayServerServices.cs index 140cc9e34..7672e94af 100644 --- a/BTCPayServer/Hosting/BTCPayServerServices.cs +++ b/BTCPayServer/Hosting/BTCPayServerServices.cs @@ -97,16 +97,15 @@ namespace BTCPayServer.Hosting services.TryAddSingleton(); services.TryAddSingleton(o => o.GetRequiredService>().Value); + // Don't move this StartupTask, we depend on it being right here services.AddStartupTask(); + // services.AddStartupTask(); services.TryAddSingleton(o => { var datadirs = o.GetRequiredService(); var dbContext = o.GetRequiredService(); - var dbpath = Path.Combine(datadirs.DataDir, "InvoiceDB"); - if (!Directory.Exists(dbpath)) - Directory.CreateDirectory(dbpath); - return new InvoiceRepository(dbContext, dbpath, o.GetRequiredService(), o.GetService()); + return new InvoiceRepository(dbContext, o.GetRequiredService(), o.GetService()); }); services.AddSingleton(); services.TryAddSingleton(); @@ -264,6 +263,8 @@ namespace BTCPayServer.Hosting services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddShopify(); #if DEBUG services.AddSingleton(); diff --git a/BTCPayServer/Hosting/MigrationStartupTask.cs b/BTCPayServer/Hosting/MigrationStartupTask.cs index 1bce3c1ea..77d48ef15 100644 --- a/BTCPayServer/Hosting/MigrationStartupTask.cs +++ b/BTCPayServer/Hosting/MigrationStartupTask.cs @@ -1,16 +1,23 @@ using System; +using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using BTCPayServer.Abstractions.Contracts; using BTCPayServer.Client.Models; +using BTCPayServer.Configuration; using BTCPayServer.Data; using BTCPayServer.Logging; using BTCPayServer.Services; using BTCPayServer.Services.Stores; +using DBriize; +using DBriize.Utils; using Microsoft.AspNetCore.Identity; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using NBitcoin.DataEncoders; namespace BTCPayServer.Hosting { @@ -20,18 +27,21 @@ namespace BTCPayServer.Hosting private readonly StoreRepository _StoreRepository; private readonly BTCPayNetworkProvider _NetworkProvider; private readonly SettingsRepository _Settings; + private readonly BTCPayServerOptions _btcPayServerOptions; private readonly UserManager _userManager; public MigrationStartupTask( BTCPayNetworkProvider networkProvider, StoreRepository storeRepository, ApplicationDbContextFactory dbContextFactory, UserManager userManager, - SettingsRepository settingsRepository) + SettingsRepository settingsRepository, + BTCPayServerOptions btcPayServerOptions) { _DBContextFactory = dbContextFactory; _StoreRepository = storeRepository; _NetworkProvider = networkProvider; _Settings = settingsRepository; + _btcPayServerOptions = btcPayServerOptions; _userManager = userManager; } public async Task ExecuteAsync(CancellationToken cancellationToken = default) diff --git a/BTCPayServer/Services/Invoices/InvoiceRepository.cs b/BTCPayServer/Services/Invoices/InvoiceRepository.cs index ca19f2fcf..455795cbb 100644 --- a/BTCPayServer/Services/Invoices/InvoiceRepository.cs +++ b/BTCPayServer/Services/Invoices/InvoiceRepository.cs @@ -2,16 +2,13 @@ using System; using System.Collections.Generic; using System.Globalization; using System.Linq; -using System.Runtime.CompilerServices; using System.Threading.Tasks; using BTCPayServer.Client.Models; using BTCPayServer.Data; using BTCPayServer.Events; using BTCPayServer.Logging; using BTCPayServer.Models.InvoicingModels; -using BTCPayServer.Models.StoreViewModels; using BTCPayServer.Payments; -using DBriize; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using NBitcoin; @@ -22,7 +19,7 @@ using InvoiceData = BTCPayServer.Data.InvoiceData; namespace BTCPayServer.Services.Invoices { - public class InvoiceRepository : IDisposable + public class InvoiceRepository { static JsonSerializerSettings DefaultSerializerSettings; static InvoiceRepository() @@ -31,31 +28,13 @@ namespace BTCPayServer.Services.Invoices NBitcoin.JsonConverters.Serializer.RegisterFrontConverters(DefaultSerializerSettings); } - private readonly DBriizeEngine _Engine; - public DBriizeEngine Engine - { - get - { - return _Engine; - } - } - private readonly ApplicationDbContextFactory _ContextFactory; private readonly EventAggregator _eventAggregator; private readonly BTCPayNetworkProvider _Networks; - private readonly CustomThreadPool _IndexerThread; - public InvoiceRepository(ApplicationDbContextFactory contextFactory, string dbreezePath, + public InvoiceRepository(ApplicationDbContextFactory contextFactory, BTCPayNetworkProvider networks, EventAggregator eventAggregator) { - int retryCount = 0; -retry: - try - { - _Engine = new DBriizeEngine(dbreezePath); - } - catch when (retryCount++ < 5) { goto retry; } - _IndexerThread = new CustomThreadPool(1, "Invoice Indexer"); _ContextFactory = contextFactory; _Networks = networks; _eventAggregator = eventAggregator; @@ -148,7 +127,7 @@ retry: if (invoiceData.CustomerEmail == null && data.Email != null) { invoiceData.CustomerEmail = data.Email; - AddToTextSearch(invoiceId, invoiceData.CustomerEmail); + AddToTextSearch(ctx, invoiceData, invoiceData.CustomerEmail); } await ctx.SaveChangesAsync().ConfigureAwait(false); } @@ -170,7 +149,7 @@ retry: public async Task CreateInvoiceAsync(string storeId, InvoiceEntity invoice) { - List textSearch = new List(); + var textSearch = new List(); invoice = Clone(invoice); invoice.Networks = _Networks; invoice.Id = Encoders.Base58.EncodeData(RandomUtils.GetBytes(16)); @@ -180,7 +159,7 @@ retry: invoice.StoreId = storeId; using (var context = _ContextFactory.CreateContext()) { - context.Invoices.Add(new Data.InvoiceData() + var invoiceData = new Data.InvoiceData() { StoreDataId = storeId, Id = invoice.Id, @@ -193,7 +172,9 @@ retry: ItemCode = invoice.Metadata.ItemCode, CustomerEmail = invoice.RefundMail, Archived = false - }); + }; + await context.Invoices.AddAsync(invoiceData); + foreach (var paymentMethod in invoice.GetPaymentMethods()) { @@ -202,13 +183,13 @@ retry: var paymentDestination = paymentMethod.GetPaymentMethodDetails().GetPaymentDestination(); string address = GetDestination(paymentMethod); - context.AddressInvoices.Add(new AddressInvoiceData() + await context.AddressInvoices.AddAsync(new AddressInvoiceData() { InvoiceDataId = invoice.Id, CreatedTime = DateTimeOffset.UtcNow, }.Set(address, paymentMethod.GetId())); - context.HistoricalAddressInvoices.Add(new HistoricalAddressInvoiceData() + await context.HistoricalAddressInvoices.AddAsync(new HistoricalAddressInvoiceData() { InvoiceDataId = invoice.Id, Assigned = DateTimeOffset.UtcNow @@ -216,18 +197,21 @@ retry: textSearch.Add(paymentDestination); textSearch.Add(paymentMethod.Calculate().TotalDue.ToString()); } - context.PendingInvoices.Add(new PendingInvoiceData() { Id = invoice.Id }); + await context.PendingInvoices.AddAsync(new PendingInvoiceData() { Id = invoice.Id }); + + textSearch.Add(invoice.Id); + textSearch.Add(invoice.InvoiceTime.ToString(CultureInfo.InvariantCulture)); + textSearch.Add(invoice.Price.ToString(CultureInfo.InvariantCulture)); + textSearch.Add(invoice.Metadata.OrderId); + textSearch.Add(ToJsonString(invoice.Metadata, null)); + textSearch.Add(invoice.StoreId); + textSearch.Add(invoice.Metadata.BuyerEmail); + AddToTextSearch(context, invoiceData, textSearch.ToArray()); + await context.SaveChangesAsync().ConfigureAwait(false); } - textSearch.Add(invoice.Id); - textSearch.Add(invoice.InvoiceTime.ToString(CultureInfo.InvariantCulture)); - textSearch.Add(invoice.Price.ToString(CultureInfo.InvariantCulture)); - textSearch.Add(invoice.Metadata.OrderId); - textSearch.Add(ToString(invoice.Metadata, null)); - textSearch.Add(invoice.StoreId); - textSearch.Add(invoice.Metadata.BuyerEmail); - AddToTextSearch(invoice.Id, textSearch.ToArray()); + return invoice; } @@ -295,10 +279,10 @@ retry: invoice.Blob = ToBytes(invoiceEntity, network); await context.AddressInvoices.AddAsync(new AddressInvoiceData() - { - InvoiceDataId = invoiceId, - CreatedTime = DateTimeOffset.UtcNow - } + { + InvoiceDataId = invoiceId, + CreatedTime = DateTimeOffset.UtcNow + } .Set(GetDestination(paymentMethod), paymentMethod.GetId())); await context.HistoricalAddressInvoices.AddAsync(new HistoricalAddressInvoiceData() { @@ -306,8 +290,8 @@ retry: Assigned = DateTimeOffset.UtcNow }.SetAddress(paymentMethodDetails.GetPaymentDestination(), network.CryptoCode)); + AddToTextSearch(context, invoice, paymentMethodDetails.GetPaymentDestination()); await context.SaveChangesAsync(); - AddToTextSearch(invoice.Id, paymentMethodDetails.GetPaymentDestination()); return true; } @@ -357,7 +341,7 @@ retry: data.UnAssigned == null); foreach (var historicalAddressInvoiceData in addresses) { - historicalAddressInvoiceData.UnAssigned = DateTimeOffset.UtcNow; + historicalAddressInvoiceData.UnAssigned = DateTimeOffset.UtcNow; } } @@ -372,29 +356,13 @@ retry: catch (DbUpdateException) { } //Possibly, it was unassigned before } - private string[] SearchInvoice(string searchTerms) + public static void AddToTextSearch(ApplicationDbContext context, InvoiceData invoice, params string[] terms) { - using (var tx = _Engine.GetTransaction()) - { - var terms = searchTerms.Split(null); - searchTerms = string.Join(' ', terms.Select(t => t.Length > 50 ? t.Substring(0, 50) : t).ToArray()); - return tx.TextSearch("InvoiceSearch").Block(searchTerms) - .GetDocumentIDs() - .Select(id => Encoders.Base58.EncodeData(id)) - .ToArray(); - } - } - - void AddToTextSearch(string invoiceId, params string[] terms) - { - _IndexerThread.DoAsync(() => - { - using (var tx = _Engine.GetTransaction()) - { - tx.TextAppend("InvoiceSearch", Encoders.Base58.DecodeData(invoiceId), string.Join(" ", terms.Where(t => !string.IsNullOrWhiteSpace(t)))); - tx.Commit(); - } - }); + var filteredTerms = terms.Where(t => !string.IsNullOrWhiteSpace(t) + && (invoice.InvoiceSearchData == null || invoice.InvoiceSearchData.All(data => data.Value != t))) + .Distinct() + .Select(s => new InvoiceSearchData() { InvoiceDataId = invoice.Id, Value = s }); + context.AddRange(filteredTerms); } public async Task UpdateInvoiceStatus(string invoiceId, InvoiceState invoiceState) @@ -415,7 +383,8 @@ retry: using (var context = _ContextFactory.CreateContext()) { var items = context.Invoices.Where(a => invoiceIds.Contains(a.Id)); - if (items == null) { + if (items == null) + { return; } @@ -423,7 +392,7 @@ retry: { invoice.Archived = true; } - + await context.SaveChangesAsync(); } } @@ -441,7 +410,7 @@ retry: await context.SaveChangesAsync().ConfigureAwait(false); } } - public async Task UpdateInvoiceMetadata(string invoiceId, string storeId, JObject metadata) + public async Task UpdateInvoiceMetadata(string invoiceId, string storeId, JObject metadata) { using (var context = _ContextFactory.CreateContext()) { @@ -451,7 +420,7 @@ retry: StringComparison.InvariantCultureIgnoreCase))) return null; var blob = invoiceData.GetBlob(_Networks); - blob.Metadata = InvoiceMetadata.FromJObject(metadata); + blob.Metadata = InvoiceMetadata.FromJObject(metadata); invoiceData.Blob = ToBytes(blob); await context.SaveChangesAsync().ConfigureAwait(false); return ToEntity(invoiceData); @@ -595,9 +564,11 @@ retry: private IQueryable GetInvoiceQuery(ApplicationDbContext context, InvoiceQuery queryObject) { - IQueryable query = queryObject.UserId is null + IQueryable query = queryObject.UserId is null ? context.Invoices - : context.UserStore.Where(u => u.ApplicationUserId == queryObject.UserId).SelectMany(c => c.StoreData.Invoices); + : context.UserStore + .Where(u => u.ApplicationUserId == queryObject.UserId) + .SelectMany(c => c.StoreData.Invoices); if (!queryObject.IncludeArchived) { @@ -618,14 +589,9 @@ retry: if (!string.IsNullOrEmpty(queryObject.TextSearch)) { - var ids = new HashSet(SearchInvoice(queryObject.TextSearch)).ToArray(); - if (ids.Length == 0) - { - // Hacky way to return an empty query object. The nice way is much too elaborate: - // https://stackoverflow.com/questions/33305495/how-to-return-empty-iqueryable-in-an-async-repository-method - return query.Where(x => false); - } - query = query.Where(i => ids.Contains(i.Id)); +#pragma warning disable CA1307 // Specify StringComparison + query = query.Where(i => i.InvoiceSearchData.Any(data => data.Value.StartsWith(queryObject.TextSearch))); +#pragma warning restore CA1307 // Specify StringComparison } if (queryObject.StartDate != null) @@ -668,8 +634,8 @@ retry: if (queryObject.Skip != null) query = query.Skip(queryObject.Skip.Value); - if (queryObject.Count != null) - query = query.Take(queryObject.Count.Value); + if (queryObject.Take != null) + query = query.Take(queryObject.Take.Value); return query; } @@ -771,12 +737,12 @@ retry: await context.Payments.AddAsync(data); + AddToTextSearch(context, invoice, paymentData.GetSearchTerms()); try { await context.SaveChangesAsync().ConfigureAwait(false); } catch (DbUpdateException) { return null; } // Already exists - AddToTextSearch(invoiceId, paymentData.GetSearchTerms()); return entity; } } @@ -802,12 +768,12 @@ retry: } } - private byte[] ToBytes(T obj, BTCPayNetworkBase network = null) + private static byte[] ToBytes(T obj, BTCPayNetworkBase network = null) { - return ZipUtils.Zip(ToString(obj, network)); + return ZipUtils.Zip(ToJsonString(obj, network)); } - private string ToString(T data, BTCPayNetworkBase network) + public static string ToJsonString(T data, BTCPayNetworkBase network) { if (network == null) { @@ -815,14 +781,6 @@ retry: } return network.ToString(data); } - - public void Dispose() - { - if (_Engine != null) - _Engine.Dispose(); - if (_IndexerThread != null) - _IndexerThread.Dispose(); - } } public class InvoiceQuery @@ -854,7 +812,7 @@ retry: get; set; } - public int? Count + public int? Take { get; set; } diff --git a/BTCPayServer/Services/MigrationSettings.cs b/BTCPayServer/Services/MigrationSettings.cs index 21cdb58f6..c81949382 100644 --- a/BTCPayServer/Services/MigrationSettings.cs +++ b/BTCPayServer/Services/MigrationSettings.cs @@ -13,5 +13,8 @@ namespace BTCPayServer.Services { return string.Empty; } + + // Done in DbMigrationsHostedService + public int? MigratedInvoiceTextSearchPages { get; set; } } } diff --git a/BTCPayServer/Services/SettingsRepository.cs b/BTCPayServer/Services/SettingsRepository.cs index bb9730fad..7f6772c08 100644 --- a/BTCPayServer/Services/SettingsRepository.cs +++ b/BTCPayServer/Services/SettingsRepository.cs @@ -30,17 +30,11 @@ namespace BTCPayServer.Services return Deserialize(data.Value); } } - public async Task UpdateSetting(T obj, string name = null) { - name ??= obj.GetType().FullName; using (var ctx = _ContextFactory.CreateContext()) { - var settings = new SettingData(); - settings.Id = name; - settings.Value = Serialize(obj); - ctx.Attach(settings); - ctx.Entry(settings).State = EntityState.Modified; + var settings = UpdateSettingInContext(ctx, obj, name); try { await ctx.SaveChangesAsync(); @@ -55,7 +49,19 @@ namespace BTCPayServer.Services { Settings = obj }); + } + public SettingData UpdateSettingInContext(ApplicationDbContext ctx, T obj, string name = null) + { + name ??= obj.GetType().FullName; + var settings = new SettingData(); + settings.Id = name; + settings.Value = Serialize(obj); + + ctx.Attach(settings); + ctx.Entry(settings).State = EntityState.Modified; + + return settings; } private T Deserialize(string value)