mirrors/btcpayserver
1
0
Fork 0
mirror of https://github.com/btcpayserver/btcpayserver.git synced 2025-02-20 13:34:37 +01:00
Code Issues Projects Releases Packages Wiki Activity

Remove only dependency on Dbriize (TextSearch in new invoice column) (#2029)

Browse source 
* Remove only dependency on Dbriize (TextSearch in new invoice column)

* Switch to table for invoice text search

* Adding missing using after rebase

* Removing database migration in preparation for refresh

* Database Migration: Adding InvoiceSearchData

* Refactoring InvoicesRepository to make AddToTextSearch static and non-async

Operation as async is too expensive for simple filtering and AddRange

* Renaming InvoiceQuery property to Take

More inline with what property does by convention, Take is used in conjuction with Skip

* Refactoring SettingsRepository so update of settings can happen in another context

* Adding DbMigrationsHostedService that performs long running data migrations

* Commenting special placing of MigrationStartupTask

* Simplifying code and leaving comment on expected flow

* Resolving problems after merge

* Database Migration: Refreshing database migration, ensuring no unintended changes on ModelSnapshot

Co-authored-by: rockstardev <rockstardev@users.noreply.github.com>
Co-authored-by: Nicolas Dorier <nicolas.dorier@gmail.com>
This commit is contained in:
Andrew Camilleri 2020-12-28 11:10:53 +01:00 • committed by GitHub
parent a6ee64ea63
commit 39b5462809
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 324 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
BTCPayServer.Data/Data/ApplicationDbContext.cs
View file

@ -67,6 +67,7 @@ namespace BTCPayServer.Data
public DbSet<WebhookData> Webhooks { get; set; }
public DbSet<WebhookDeliveryData> WebhookDeliveries { get; set; }
public DbSet<InvoiceWebhookDeliveryData> InvoiceWebhookDeliveries { get; set; }
public DbSet<InvoiceSearchData> InvoiceSearchDatas { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
@ -81,6 +82,7 @@ namespace BTCPayServer.Data
Data.UserStore.OnModelCreating(builder);
NotificationData.OnModelCreating(builder);
InvoiceData.OnModelCreating(builder);
InvoiceSearchData.OnModelCreating(builder);
PaymentData.OnModelCreating(builder);
Data.UserStore.OnModelCreating(builder);
APIKeyData.OnModelCreating(builder);

1
BTCPayServer.Data/Data/InvoiceData.cs
View file

@ -75,6 +75,7 @@ namespace BTCPayServer.Data
}
public bool Archived { get; set; }
public List<PendingInvoiceData> PendingInvoices { get; set; }
public List<InvoiceSearchData> InvoiceSearchData { get; set; }
public List<RefundData> Refunds { get; set; }
public string CurrentRefundId { get; set; }
[ForeignKey("Id,CurrentRefundId")]

38
BTCPayServer.Data/Data/InvoiceSearchData.cs Normal file
View file

@ -0,0 +1,38 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
namespace BTCPayServer.Data
{
public class InvoiceSearchData
{
[DatabaseGenerated(DatabaseGeneratedOption.Identity)]
[Key]
public int Id { get; set; }
[ForeignKey(nameof(InvoiceData))]
public string InvoiceDataId { get; set; }
public InvoiceData InvoiceData { get; set; }
public string Value { get; set; }
internal static void OnModelCreating(ModelBuilder builder)
{
builder.Entity<InvoiceSearchData>()
.HasOne(o => o.InvoiceData)
.WithMany(a => a.InvoiceSearchData)
.OnDelete(DeleteBehavior.Cascade);
builder.Entity<InvoiceSearchData>()
.HasIndex(data => data.Value);
builder.Entity<InvoiceSearchData>()
.Property(a => a.Id)
.HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn)
.HasAnnotation("MySql:ValueGeneratedOnAdd", true)
.HasAnnotation("Sqlite:Autoincrement", true);
}
}
}

55
BTCPayServer.Data/Migrations/20201227165824_AdddingInvoiceSearchData.cs Normal file
View file

@ -0,0 +1,55 @@
using BTCPayServer.Data;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
namespace BTCPayServer.Migrations
{
[DbContext(typeof(ApplicationDbContext))]
[Migration("20201227165824_AdddingInvoiceSearchData")]
public partial class AdddingInvoiceSearchData : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "InvoiceSearchDatas",
columns: table => new
{
Id = table.Column<int>(nullable: false)
// manually added
.Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn)
.Annotation("MySql:ValueGeneratedOnAdd", true)
.Annotation("Sqlite:Autoincrement", true),
// eof manually added
InvoiceDataId = table.Column<string>(nullable: true),
Value = table.Column<string>(nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_InvoiceSearchDatas", x => x.Id);
table.ForeignKey(
name: "FK_InvoiceSearchDatas_Invoices_InvoiceDataId",
column: x => x.InvoiceDataId,
principalTable: "Invoices",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "IX_InvoiceSearchDatas_InvoiceDataId",
table: "InvoiceSearchDatas",
column: "InvoiceDataId");
migrationBuilder.CreateIndex(
name: "IX_InvoiceSearchDatas_Value",
table: "InvoiceSearchDatas",
column: "Value");
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "InvoiceSearchDatas");
}
}
}

33
BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs
View file

@ -4,6 +4,7 @@ using BTCPayServer.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
namespace BTCPayServer.Migrations
{
@ -259,6 +260,30 @@ namespace BTCPayServer.Migrations
b.ToTable("InvoiceEvents");
});
modelBuilder.Entity("BTCPayServer.Data.InvoiceSearchData", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasAnnotation("MySql:ValueGeneratedOnAdd", true)
.HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn)
.HasAnnotation("Sqlite:Autoincrement", true);
b.Property<string>("InvoiceDataId")
.HasColumnType("TEXT");
b.Property<string>("Value")
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("InvoiceDataId");
b.HasIndex("Value");
b.ToTable("InvoiceSearchDatas");
});
modelBuilder.Entity("BTCPayServer.Data.InvoiceWebhookDeliveryData", b =>
{
b.Property<string>("InvoiceId")
@ -963,6 +988,14 @@ namespace BTCPayServer.Migrations
.IsRequired();
});
modelBuilder.Entity("BTCPayServer.Data.InvoiceSearchData", b =>
{
b.HasOne("BTCPayServer.Data.InvoiceData", "InvoiceData")
.WithMany("InvoiceSearchData")
.HasForeignKey("InvoiceDataId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("BTCPayServer.Data.InvoiceWebhookDeliveryData", b =>
{
b.HasOne("BTCPayServer.Data.WebhookDeliveryData", "Delivery")

2
BTCPayServer/Controllers/InvoiceController.API.cs
View file

@ -69,7 +69,7 @@ namespace BTCPayServer.Controllers
var query = new InvoiceQuery()
{
Count = limit,
Take = limit,
Skip = offset,
EndDate = dateEnd,
StartDate = dateStart,

4
BTCPayServer/Controllers/InvoiceController.UI.cs
View file

@ -705,7 +705,7 @@ namespace BTCPayServer.Controllers
InvoiceQuery invoiceQuery = GetInvoiceQuery(model.SearchTerm, model.TimezoneOffset ?? 0);
var counting = _InvoiceRepository.GetInvoicesTotal(invoiceQuery);
invoiceQuery.Count = model.Count;
invoiceQuery.Take = model.Count;
invoiceQuery.Skip = model.Skip;
var list = await _InvoiceRepository.GetInvoices(invoiceQuery);
@ -759,7 +759,7 @@ namespace BTCPayServer.Controllers
InvoiceQuery invoiceQuery = GetInvoiceQuery(searchTerm, timezoneOffset);
invoiceQuery.Skip = 0;
invoiceQuery.Count = int.MaxValue;
invoiceQuery.Take = int.MaxValue;
var invoices = await _InvoiceRepository.GetInvoices(invoiceQuery);
var res = model.Process(invoices, format);

107
BTCPayServer/HostedServices/DbMigrationsHostedService.cs Normal file
View file

@ -0,0 +1,107 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Threading.Tasks;
using BTCPayServer.Data;
using BTCPayServer.Services;
using BTCPayServer.Services.Invoices;
namespace BTCPayServer.HostedServices
{
/// <summary>
/// In charge of all long running db migrations that we can't execute on startup in MigrationStartupTask
/// </summary>
public class DbMigrationsHostedService : BaseAsyncService
{
private readonly InvoiceRepository _invoiceRepository;
private readonly SettingsRepository _settingsRepository;
private readonly ApplicationDbContextFactory _dbContextFactory;
public DbMigrationsHostedService(InvoiceRepository invoiceRepository, SettingsRepository settingsRepository, ApplicationDbContextFactory dbContextFactory)
{
_invoiceRepository = invoiceRepository;
_settingsRepository = settingsRepository;
_dbContextFactory = dbContextFactory;
}
internal override Task[] InitializeTasks()
{
return new Task[] { ProcessMigration() };
}
protected async Task ProcessMigration()
{
var settings = await _settingsRepository.GetSettingAsync<MigrationSettings>();
if (settings.MigratedInvoiceTextSearchPages != int.MaxValue)
{
await MigratedInvoiceTextSearchToDb(settings.MigratedInvoiceTextSearchPages.Value);
}
// Refresh settings since these operations may run for very long time
}
private async Task MigratedInvoiceTextSearchToDb(int startFromPage)
{
using var ctx = _dbContextFactory.CreateContext();
var invoiceQuery = new InvoiceQuery { IncludeArchived = true };
var totalCount = await _invoiceRepository.GetInvoicesTotal(invoiceQuery);
const int PAGE_SIZE = 1000;
var totalPages = Math.Ceiling(totalCount * 1.0m / PAGE_SIZE);
for (int i = startFromPage; i < totalPages; i++)
{
invoiceQuery.Skip = i * PAGE_SIZE;
invoiceQuery.Take = PAGE_SIZE;
var invoices = await _invoiceRepository.GetInvoices(invoiceQuery);
foreach (var invoice in invoices)
{
var textSearch = new List<string>();
// recreating different textSearch.Adds that were previously in DBriize
foreach (var paymentMethod in invoice.GetPaymentMethods())
{
if (paymentMethod.Network != null)
{
var paymentDestination = paymentMethod.GetPaymentMethodDetails().GetPaymentDestination();
textSearch.Add(paymentDestination);
textSearch.Add(paymentMethod.Calculate().TotalDue.ToString());
}
}
//
textSearch.Add(invoice.Id);
textSearch.Add(invoice.InvoiceTime.ToString(CultureInfo.InvariantCulture));
textSearch.Add(invoice.Price.ToString(CultureInfo.InvariantCulture));
textSearch.Add(invoice.Metadata.OrderId);
textSearch.Add(InvoiceRepository.ToJsonString(invoice.Metadata, null));
textSearch.Add(invoice.StoreId);
textSearch.Add(invoice.Metadata.BuyerEmail);
//
textSearch.Add(invoice.RefundMail);
// TODO: Are there more things to cache? PaymentData?
InvoiceRepository.AddToTextSearch(ctx,
new InvoiceData { Id = invoice.Id, InvoiceSearchData = new List<InvoiceSearchData>() },
textSearch.ToArray());
}
var settings = await _settingsRepository.GetSettingAsync<MigrationSettings>();
if (i + 1 < totalPages)
{
settings.MigratedInvoiceTextSearchPages = i;
}
else
{
// during final pass we set int.MaxValue so migration doesn't run again
settings.MigratedInvoiceTextSearchPages = int.MaxValue;
}
// this call triggers update; we're sure that MigrationSettings is already initialized in db
// because of logic executed in MigrationStartupTask.cs
_settingsRepository.UpdateSettingInContext(ctx, settings);
await ctx.SaveChangesAsync();
}
}
}
}

9
BTCPayServer/Hosting/BTCPayServerServices.cs
View file

@ -97,16 +97,15 @@ namespace BTCPayServer.Hosting
services.TryAddSingleton<InvoicePaymentNotification>();
services.TryAddSingleton<BTCPayServerOptions>(o =>
o.GetRequiredService<IOptions<BTCPayServerOptions>>().Value);
// Don't move this StartupTask, we depend on it being right here
services.AddStartupTask<MigrationStartupTask>();
//
services.AddStartupTask<BlockExplorerLinkStartupTask>();
services.TryAddSingleton<InvoiceRepository>(o =>
{
var datadirs = o.GetRequiredService<DataDirectories>();
var dbContext = o.GetRequiredService<ApplicationDbContextFactory>();
var dbpath = Path.Combine(datadirs.DataDir, "InvoiceDB");
if (!Directory.Exists(dbpath))
Directory.CreateDirectory(dbpath);
return new InvoiceRepository(dbContext, dbpath, o.GetRequiredService<BTCPayNetworkProvider>(), o.GetService<EventAggregator>());
return new InvoiceRepository(dbContext, o.GetRequiredService<BTCPayNetworkProvider>(), o.GetService<EventAggregator>());
});
services.AddSingleton<BTCPayServerEnvironment>();
services.TryAddSingleton<TokenRepository>();
@ -264,6 +263,8 @@ namespace BTCPayServer.Hosting
services.AddSingleton<INotificationHandler, InvoiceEventNotification.Handler>();
services.AddSingleton<INotificationHandler, PayoutNotification.Handler>();
services.AddSingleton<IHostedService, DbMigrationsHostedService>();
services.AddShopify();
#if DEBUG
services.AddSingleton<INotificationHandler, JunkNotification.Handler>();

12
BTCPayServer/Hosting/MigrationStartupTask.cs
View file

@ -1,16 +1,23 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using BTCPayServer.Abstractions.Contracts;
using BTCPayServer.Client.Models;
using BTCPayServer.Configuration;
using BTCPayServer.Data;
using BTCPayServer.Logging;
using BTCPayServer.Services;
using BTCPayServer.Services.Stores;
using DBriize;
using DBriize.Utils;
using Microsoft.AspNetCore.Identity;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using NBitcoin.DataEncoders;
namespace BTCPayServer.Hosting
{
@ -20,18 +27,21 @@ namespace BTCPayServer.Hosting
private readonly StoreRepository _StoreRepository;
private readonly BTCPayNetworkProvider _NetworkProvider;
private readonly SettingsRepository _Settings;
private readonly BTCPayServerOptions _btcPayServerOptions;
private readonly UserManager<ApplicationUser> _userManager;
public MigrationStartupTask(
BTCPayNetworkProvider networkProvider,
StoreRepository storeRepository,
ApplicationDbContextFactory dbContextFactory,
UserManager<ApplicationUser> userManager,
SettingsRepository settingsRepository)
SettingsRepository settingsRepository,
BTCPayServerOptions btcPayServerOptions)
{
_DBContextFactory = dbContextFactory;
_StoreRepository = storeRepository;
_NetworkProvider = networkProvider;
_Settings = settingsRepository;
_btcPayServerOptions = btcPayServerOptions;
_userManager = userManager;
}
public async Task ExecuteAsync(CancellationToken cancellationToken = default)

148
BTCPayServer/Services/Invoices/InvoiceRepository.cs
View file

@ -2,16 +2,13 @@ using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using BTCPayServer.Client.Models;
using BTCPayServer.Data;
using BTCPayServer.Events;
using BTCPayServer.Logging;
using BTCPayServer.Models.InvoicingModels;
using BTCPayServer.Models.StoreViewModels;
using BTCPayServer.Payments;
using DBriize;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using NBitcoin;
@ -22,7 +19,7 @@ using InvoiceData = BTCPayServer.Data.InvoiceData;
namespace BTCPayServer.Services.Invoices
{
public class InvoiceRepository : IDisposable
public class InvoiceRepository
{
static JsonSerializerSettings DefaultSerializerSettings;
static InvoiceRepository()
@ -31,31 +28,13 @@ namespace BTCPayServer.Services.Invoices
NBitcoin.JsonConverters.Serializer.RegisterFrontConverters(DefaultSerializerSettings);
}
private readonly DBriizeEngine _Engine;
public DBriizeEngine Engine
{
get
{
return _Engine;
}
}
private readonly ApplicationDbContextFactory _ContextFactory;
private readonly EventAggregator _eventAggregator;
private readonly BTCPayNetworkProvider _Networks;
private readonly CustomThreadPool _IndexerThread;
public InvoiceRepository(ApplicationDbContextFactory contextFactory, string dbreezePath,
public InvoiceRepository(ApplicationDbContextFactory contextFactory,
BTCPayNetworkProvider networks, EventAggregator eventAggregator)
{
int retryCount = 0;
retry:
try
{
_Engine = new DBriizeEngine(dbreezePath);
}
catch when (retryCount++ < 5) { goto retry; }
_IndexerThread = new CustomThreadPool(1, "Invoice Indexer");
_ContextFactory = contextFactory;
_Networks = networks;
_eventAggregator = eventAggregator;
@ -148,7 +127,7 @@ retry:
if (invoiceData.CustomerEmail == null && data.Email != null)
{
invoiceData.CustomerEmail = data.Email;
AddToTextSearch(invoiceId, invoiceData.CustomerEmail);
AddToTextSearch(ctx, invoiceData, invoiceData.CustomerEmail);
}
await ctx.SaveChangesAsync().ConfigureAwait(false);
}
@ -170,7 +149,7 @@ retry:
public async Task<InvoiceEntity> CreateInvoiceAsync(string storeId, InvoiceEntity invoice)
{
List<string> textSearch = new List<string>();
var textSearch = new List<string>();
invoice = Clone(invoice);
invoice.Networks = _Networks;
invoice.Id = Encoders.Base58.EncodeData(RandomUtils.GetBytes(16));
@ -180,7 +159,7 @@ retry:
invoice.StoreId = storeId;
using (var context = _ContextFactory.CreateContext())
{
context.Invoices.Add(new Data.InvoiceData()
var invoiceData = new Data.InvoiceData()
{
StoreDataId = storeId,
Id = invoice.Id,
@ -193,7 +172,9 @@ retry:
ItemCode = invoice.Metadata.ItemCode,
CustomerEmail = invoice.RefundMail,
Archived = false
});
};
await context.Invoices.AddAsync(invoiceData);
foreach (var paymentMethod in invoice.GetPaymentMethods())
{
@ -202,13 +183,13 @@ retry:
var paymentDestination = paymentMethod.GetPaymentMethodDetails().GetPaymentDestination();
string address = GetDestination(paymentMethod);
context.AddressInvoices.Add(new AddressInvoiceData()
await context.AddressInvoices.AddAsync(new AddressInvoiceData()
{
InvoiceDataId = invoice.Id,
CreatedTime = DateTimeOffset.UtcNow,
}.Set(address, paymentMethod.GetId()));
context.HistoricalAddressInvoices.Add(new HistoricalAddressInvoiceData()
await context.HistoricalAddressInvoices.AddAsync(new HistoricalAddressInvoiceData()
{
InvoiceDataId = invoice.Id,
Assigned = DateTimeOffset.UtcNow
@ -216,18 +197,21 @@ retry:
textSearch.Add(paymentDestination);
textSearch.Add(paymentMethod.Calculate().TotalDue.ToString());
}
context.PendingInvoices.Add(new PendingInvoiceData() { Id = invoice.Id });
await context.PendingInvoices.AddAsync(new PendingInvoiceData() { Id = invoice.Id });
textSearch.Add(invoice.Id);
textSearch.Add(invoice.InvoiceTime.ToString(CultureInfo.InvariantCulture));
textSearch.Add(invoice.Price.ToString(CultureInfo.InvariantCulture));
textSearch.Add(invoice.Metadata.OrderId);
textSearch.Add(ToJsonString(invoice.Metadata, null));
textSearch.Add(invoice.StoreId);
textSearch.Add(invoice.Metadata.BuyerEmail);
AddToTextSearch(context, invoiceData, textSearch.ToArray());
await context.SaveChangesAsync().ConfigureAwait(false);
}
textSearch.Add(invoice.Id);
textSearch.Add(invoice.InvoiceTime.ToString(CultureInfo.InvariantCulture));
textSearch.Add(invoice.Price.ToString(CultureInfo.InvariantCulture));
textSearch.Add(invoice.Metadata.OrderId);
textSearch.Add(ToString(invoice.Metadata, null));
textSearch.Add(invoice.StoreId);
textSearch.Add(invoice.Metadata.BuyerEmail);
AddToTextSearch(invoice.Id, textSearch.ToArray());
return invoice;
}
@ -295,10 +279,10 @@ retry:
invoice.Blob = ToBytes(invoiceEntity, network);
await context.AddressInvoices.AddAsync(new AddressInvoiceData()
{
InvoiceDataId = invoiceId,
CreatedTime = DateTimeOffset.UtcNow
}
{
InvoiceDataId = invoiceId,
CreatedTime = DateTimeOffset.UtcNow
}
.Set(GetDestination(paymentMethod), paymentMethod.GetId()));
await context.HistoricalAddressInvoices.AddAsync(new HistoricalAddressInvoiceData()
{
@ -306,8 +290,8 @@ retry:
Assigned = DateTimeOffset.UtcNow
}.SetAddress(paymentMethodDetails.GetPaymentDestination(), network.CryptoCode));
AddToTextSearch(context, invoice, paymentMethodDetails.GetPaymentDestination());
await context.SaveChangesAsync();
AddToTextSearch(invoice.Id, paymentMethodDetails.GetPaymentDestination());
return true;
}
@ -357,7 +341,7 @@ retry:
data.UnAssigned == null);
foreach (var historicalAddressInvoiceData in addresses)
{
historicalAddressInvoiceData.UnAssigned = DateTimeOffset.UtcNow;
historicalAddressInvoiceData.UnAssigned = DateTimeOffset.UtcNow;
}
}
@ -372,29 +356,13 @@ retry:
catch (DbUpdateException) { } //Possibly, it was unassigned before
}
private string[] SearchInvoice(string searchTerms)
public static void AddToTextSearch(ApplicationDbContext context, InvoiceData invoice, params string[] terms)
{
using (var tx = _Engine.GetTransaction())
{
var terms = searchTerms.Split(null);
searchTerms = string.Join(' ', terms.Select(t => t.Length > 50 ? t.Substring(0, 50) : t).ToArray());
return tx.TextSearch("InvoiceSearch").Block(searchTerms)
.GetDocumentIDs()
.Select(id => Encoders.Base58.EncodeData(id))
.ToArray();
}
}
void AddToTextSearch(string invoiceId, params string[] terms)
{
_IndexerThread.DoAsync(() =>
{
using (var tx = _Engine.GetTransaction())
{
tx.TextAppend("InvoiceSearch", Encoders.Base58.DecodeData(invoiceId), string.Join(" ", terms.Where(t => !string.IsNullOrWhiteSpace(t))));
tx.Commit();
}
});
var filteredTerms = terms.Where(t => !string.IsNullOrWhiteSpace(t)
&& (invoice.InvoiceSearchData == null || invoice.InvoiceSearchData.All(data => data.Value != t)))
.Distinct()
.Select(s => new InvoiceSearchData() { InvoiceDataId = invoice.Id, Value = s });
context.AddRange(filteredTerms);
}
public async Task UpdateInvoiceStatus(string invoiceId, InvoiceState invoiceState)
@ -415,7 +383,8 @@ retry:
using (var context = _ContextFactory.CreateContext())
{
var items = context.Invoices.Where(a => invoiceIds.Contains(a.Id));
if (items == null) {
if (items == null)
{
return;
}
@ -423,7 +392,7 @@ retry:
{
invoice.Archived = true;
}
await context.SaveChangesAsync();
}
}
@ -441,7 +410,7 @@ retry:
await context.SaveChangesAsync().ConfigureAwait(false);
}
}
public async Task<InvoiceEntity> UpdateInvoiceMetadata(string invoiceId, string storeId, JObject metadata)
public async Task<InvoiceEntity> UpdateInvoiceMetadata(string invoiceId, string storeId, JObject metadata)
{
using (var context = _ContextFactory.CreateContext())
{
@ -451,7 +420,7 @@ retry:
StringComparison.InvariantCultureIgnoreCase)))
return null;
var blob = invoiceData.GetBlob(_Networks);
blob.Metadata = InvoiceMetadata.FromJObject(metadata);
blob.Metadata = InvoiceMetadata.FromJObject(metadata);
invoiceData.Blob = ToBytes(blob);
await context.SaveChangesAsync().ConfigureAwait(false);
return ToEntity(invoiceData);
@ -595,9 +564,11 @@ retry:
private IQueryable<Data.InvoiceData> GetInvoiceQuery(ApplicationDbContext context, InvoiceQuery queryObject)
{
IQueryable<Data.InvoiceData> query = queryObject.UserId is null
IQueryable<Data.InvoiceData> query = queryObject.UserId is null
? context.Invoices
: context.UserStore.Where(u => u.ApplicationUserId == queryObject.UserId).SelectMany(c => c.StoreData.Invoices);
: context.UserStore
.Where(u => u.ApplicationUserId == queryObject.UserId)
.SelectMany(c => c.StoreData.Invoices);
if (!queryObject.IncludeArchived)
{
@ -618,14 +589,9 @@ retry:
if (!string.IsNullOrEmpty(queryObject.TextSearch))
{
var ids = new HashSet<string>(SearchInvoice(queryObject.TextSearch)).ToArray();
if (ids.Length == 0)
{
// Hacky way to return an empty query object. The nice way is much too elaborate:
// https://stackoverflow.com/questions/33305495/how-to-return-empty-iqueryable-in-an-async-repository-method
return query.Where(x => false);
}
query = query.Where(i => ids.Contains(i.Id));
#pragma warning disable CA1307 // Specify StringComparison
query = query.Where(i => i.InvoiceSearchData.Any(data => data.Value.StartsWith(queryObject.TextSearch)));
#pragma warning restore CA1307 // Specify StringComparison
}
if (queryObject.StartDate != null)
@ -668,8 +634,8 @@ retry:
if (queryObject.Skip != null)
query = query.Skip(queryObject.Skip.Value);
if (queryObject.Count != null)
query = query.Take(queryObject.Count.Value);
if (queryObject.Take != null)
query = query.Take(queryObject.Take.Value);
return query;
}
@ -771,12 +737,12 @@ retry:
await context.Payments.AddAsync(data);
AddToTextSearch(context, invoice, paymentData.GetSearchTerms());
try
{
await context.SaveChangesAsync().ConfigureAwait(false);
}
catch (DbUpdateException) { return null; } // Already exists
AddToTextSearch(invoiceId, paymentData.GetSearchTerms());
return entity;
}
}
@ -802,12 +768,12 @@ retry:
}
}
private byte[] ToBytes<T>(T obj, BTCPayNetworkBase network = null)
private static byte[] ToBytes<T>(T obj, BTCPayNetworkBase network = null)
{
return ZipUtils.Zip(ToString(obj, network));
return ZipUtils.Zip(ToJsonString(obj, network));
}
private string ToString<T>(T data, BTCPayNetworkBase network)
public static string ToJsonString<T>(T data, BTCPayNetworkBase network)
{
if (network == null)
{
@ -815,14 +781,6 @@ retry:
}
return network.ToString(data);
}
public void Dispose()
{
if (_Engine != null)
_Engine.Dispose();
if (_IndexerThread != null)
_IndexerThread.Dispose();
}
}
public class InvoiceQuery
@ -854,7 +812,7 @@ retry:
get; set;
}
public int? Count
public int? Take
{
get; set;
}

3
BTCPayServer/Services/MigrationSettings.cs
View file

@ -13,5 +13,8 @@ namespace BTCPayServer.Services
{
return string.Empty;
}
// Done in DbMigrationsHostedService
public int? MigratedInvoiceTextSearchPages { get; set; }
}
}

20
BTCPayServer/Services/SettingsRepository.cs
View file

@ -30,17 +30,11 @@ namespace BTCPayServer.Services
return Deserialize<T>(data.Value);
}
}
public async Task UpdateSetting<T>(T obj, string name = null)
{
name ??= obj.GetType().FullName;
using (var ctx = _ContextFactory.CreateContext())
{
var settings = new SettingData();
settings.Id = name;
settings.Value = Serialize(obj);
ctx.Attach(settings);
ctx.Entry(settings).State = EntityState.Modified;
var settings = UpdateSettingInContext<T>(ctx, obj, name);
try
{
await ctx.SaveChangesAsync();
@ -55,7 +49,19 @@ namespace BTCPayServer.Services
{
Settings = obj
});
}
public SettingData UpdateSettingInContext<T>(ApplicationDbContext ctx, T obj, string name = null)
{
name ??= obj.GetType().FullName;
var settings = new SettingData();
settings.Id = name;
settings.Value = Serialize(obj);
ctx.Attach(settings);
ctx.Entry(settings).State = EntityState.Modified;
return settings;
}
private T Deserialize<T>(string value)