using DBreeze; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Text; using NBitpayClient; using Newtonsoft.Json; using System.Linq; using NBitcoin; using NBitcoin.DataEncoders; using Microsoft.AspNetCore.Identity.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Query; using Microsoft.EntityFrameworkCore.Infrastructure.Internal; using BTCPayServer.Models; using System.Threading.Tasks; using BTCPayServer.Data; using System.Globalization; using BTCPayServer.Models.InvoicingModels; using BTCPayServer.Logging; using BTCPayServer.Payments; using System.Data.Common; namespace BTCPayServer.Services.Invoices { public class InvoiceRepository : IDisposable { private readonly DBreezeEngine _Engine; public DBreezeEngine Engine { get { return _Engine; } } private ApplicationDbContextFactory _ContextFactory; private CustomThreadPool _IndexerThread; public InvoiceRepository(ApplicationDbContextFactory contextFactory, string dbreezePath) { int retryCount = 0; retry: try { _Engine = new DBreezeEngine(dbreezePath); } catch when (retryCount++ < 5) { goto retry; } _IndexerThread = new CustomThreadPool(1, "Invoice Indexer"); _ContextFactory = contextFactory; } public async Task RemovePendingInvoice(string invoiceId) { Logs.PayServer.LogInformation($"Remove pending invoice {invoiceId}"); using (var ctx = _ContextFactory.CreateContext()) { ctx.PendingInvoices.Remove(new PendingInvoiceData() { Id = invoiceId }); try { await ctx.SaveChangesAsync(); return true; } catch (DbUpdateException) { return false; } } } public async Task GetInvoiceFromScriptPubKey(Script scriptPubKey, string cryptoCode) { using (var db = _ContextFactory.CreateContext()) { var key = scriptPubKey.Hash.ToString() + "#" + cryptoCode; var result = await db.AddressInvoices #pragma warning disable CS0618 .Where(a => a.Address == key) #pragma warning restore CS0618 .Select(a => a.InvoiceData) .Include(a => a.Payments) .Include(a => a.RefundAddresses) .FirstOrDefaultAsync(); if (result == null) return null; return ToEntity(result); } } public async Task GetPendingInvoices() { using (var ctx = _ContextFactory.CreateContext()) { return await ctx.PendingInvoices.Select(p => p.Id).ToArrayAsync(); } } public async Task GetAppsTaggingStore(string storeId) { if (storeId == null) throw new ArgumentNullException(nameof(storeId)); using (var ctx = _ContextFactory.CreateContext()) { return await ctx.Apps.Where(a => a.StoreDataId == storeId && a.TagAllInvoices).ToArrayAsync(); } } public async Task UpdateInvoice(string invoiceId, UpdateCustomerModel data) { using (var ctx = _ContextFactory.CreateContext()) { var invoiceData = await ctx.Invoices.FindAsync(invoiceId).ConfigureAwait(false); if (invoiceData == null) return; if (invoiceData.CustomerEmail == null && data.Email != null) { invoiceData.CustomerEmail = data.Email; } await ctx.SaveChangesAsync().ConfigureAwait(false); } } public async Task CreateInvoiceAsync(string storeId, InvoiceEntity invoice, InvoiceLogs creationLogs, BTCPayNetworkProvider networkProvider) { List textSearch = new List(); invoice = Clone(invoice, null); invoice.Id = Encoders.Base58.EncodeData(RandomUtils.GetBytes(16)); #pragma warning disable CS0618 invoice.Payments = new List(); #pragma warning restore CS0618 invoice.StoreId = storeId; using (var context = _ContextFactory.CreateContext()) { context.Invoices.Add(new Data.InvoiceData() { StoreDataId = storeId, Id = invoice.Id, Created = invoice.InvoiceTime, Blob = ToBytes(invoice, null), OrderId = invoice.OrderId, #pragma warning disable CS0618 // Type or member is obsolete Status = invoice.StatusString, #pragma warning restore CS0618 // Type or member is obsolete ItemCode = invoice.ProductInformation.ItemCode, CustomerEmail = invoice.RefundMail }); foreach (var paymentMethod in invoice.GetPaymentMethods(networkProvider)) { if (paymentMethod.Network == null) throw new InvalidOperationException("CryptoCode unsupported"); var paymentDestination = paymentMethod.GetPaymentMethodDetails().GetPaymentDestination(); string address = GetDestination(paymentMethod, paymentMethod.Network.NBitcoinNetwork); context.AddressInvoices.Add(new AddressInvoiceData() { InvoiceDataId = invoice.Id, CreatedTime = DateTimeOffset.UtcNow, }.Set(address, paymentMethod.GetId())); context.HistoricalAddressInvoices.Add(new HistoricalAddressInvoiceData() { InvoiceDataId = invoice.Id, Assigned = DateTimeOffset.UtcNow }.SetAddress(paymentDestination, paymentMethod.GetId().ToString())); textSearch.Add(paymentDestination); textSearch.Add(paymentMethod.Calculate().TotalDue.ToString()); } context.PendingInvoices.Add(new PendingInvoiceData() { Id = invoice.Id }); foreach (var log in creationLogs.ToList()) { context.InvoiceEvents.Add(new InvoiceEventData() { InvoiceDataId = invoice.Id, Message = log.Log, Timestamp = log.Timestamp, UniqueId = Encoders.Hex.EncodeData(RandomUtils.GetBytes(10)) }); } await context.SaveChangesAsync().ConfigureAwait(false); } textSearch.Add(invoice.Id); textSearch.Add(invoice.InvoiceTime.ToString(CultureInfo.InvariantCulture)); textSearch.Add(invoice.ProductInformation.Price.ToString(CultureInfo.InvariantCulture)); textSearch.Add(invoice.OrderId); textSearch.Add(ToString(invoice.BuyerInformation, null)); textSearch.Add(ToString(invoice.ProductInformation, null)); textSearch.Add(invoice.StoreId); AddToTextSearch(invoice.Id, textSearch.ToArray()); return invoice; } private static string GetDestination(PaymentMethod paymentMethod, Network network) { // For legacy reason, BitcoinLikeOnChain is putting the hashes of addresses in database if (paymentMethod.GetId().PaymentType == Payments.PaymentTypes.BTCLike) { return ((Payments.Bitcoin.BitcoinLikeOnChainPaymentMethod)paymentMethod.GetPaymentMethodDetails()).GetDepositAddress(network).ScriptPubKey.Hash.ToString(); } /////////////// return paymentMethod.GetPaymentMethodDetails().GetPaymentDestination(); } public async Task NewAddress(string invoiceId, Payments.Bitcoin.BitcoinLikeOnChainPaymentMethod paymentMethod, BTCPayNetwork network) { using (var context = _ContextFactory.CreateContext()) { var invoice = await context.Invoices.FirstOrDefaultAsync(i => i.Id == invoiceId); if (invoice == null) return false; var invoiceEntity = ToObject(invoice.Blob, network.NBitcoinNetwork); var currencyData = invoiceEntity.GetPaymentMethod(network, paymentMethod.GetPaymentType(), null); if (currencyData == null) return false; var existingPaymentMethod = (Payments.Bitcoin.BitcoinLikeOnChainPaymentMethod)currencyData.GetPaymentMethodDetails(); if (existingPaymentMethod.GetPaymentDestination() != null) { MarkUnassigned(invoiceId, invoiceEntity, context, currencyData.GetId()); } existingPaymentMethod.SetPaymentDestination(paymentMethod.GetPaymentDestination()); currencyData.SetPaymentMethodDetails(existingPaymentMethod); #pragma warning disable CS0618 if (network.IsBTC) { invoiceEntity.DepositAddress = currencyData.DepositAddress; } #pragma warning restore CS0618 invoiceEntity.SetPaymentMethod(currencyData); invoice.Blob = ToBytes(invoiceEntity, network.NBitcoinNetwork); context.AddressInvoices.Add(new AddressInvoiceData() { InvoiceDataId = invoiceId, CreatedTime = DateTimeOffset.UtcNow } .Set(GetDestination(currencyData, network.NBitcoinNetwork), currencyData.GetId())); context.HistoricalAddressInvoices.Add(new HistoricalAddressInvoiceData() { InvoiceDataId = invoiceId, Assigned = DateTimeOffset.UtcNow }.SetAddress(paymentMethod.GetPaymentDestination(), network.CryptoCode)); await context.SaveChangesAsync(); AddToTextSearch(invoice.Id, paymentMethod.GetPaymentDestination()); return true; } } public async Task AddInvoiceEvent(string invoiceId, object evt) { using (var context = _ContextFactory.CreateContext()) { context.InvoiceEvents.Add(new InvoiceEventData() { InvoiceDataId = invoiceId, Message = evt.ToString(), Timestamp = DateTimeOffset.UtcNow, UniqueId = Encoders.Hex.EncodeData(RandomUtils.GetBytes(10)) }); try { await context.SaveChangesAsync(); } catch (DbUpdateException) { } // Probably the invoice does not exists anymore } } private static void MarkUnassigned(string invoiceId, InvoiceEntity entity, ApplicationDbContext context, PaymentMethodId paymentMethodId) { foreach (var address in entity.GetPaymentMethods(null)) { if (paymentMethodId != null && paymentMethodId != address.GetId()) continue; var historical = new HistoricalAddressInvoiceData(); historical.InvoiceDataId = invoiceId; historical.SetAddress(address.GetPaymentMethodDetails().GetPaymentDestination(), address.GetId().ToString()); historical.UnAssigned = DateTimeOffset.UtcNow; context.Attach(historical); context.Entry(historical).Property(o => o.UnAssigned).IsModified = true; } } public async Task UnaffectAddress(string invoiceId) { using (var context = _ContextFactory.CreateContext()) { var invoiceData = await context.FindAsync(invoiceId).ConfigureAwait(false); if (invoiceData == null) return; var invoiceEntity = ToObject(invoiceData.Blob, null); MarkUnassigned(invoiceId, invoiceEntity, context, null); try { await context.SaveChangesAsync(); } catch (DbUpdateException) { } //Possibly, it was unassigned before } } private string[] SearchInvoice(string searchTerms) { using (var tx = _Engine.GetTransaction()) { var terms = searchTerms.Split(null); searchTerms = string.Join(' ', terms.Select(t => t.Length > 50 ? t.Substring(0, 50) : t).ToArray()); return tx.TextSearch("InvoiceSearch").Block(searchTerms) .GetDocumentIDs() .Select(id => Encoders.Base58.EncodeData(id)) .ToArray(); } } void AddToTextSearch(string invoiceId, params string[] terms) { _IndexerThread.DoAsync(() => { using (var tx = _Engine.GetTransaction()) { tx.TextAppend("InvoiceSearch", Encoders.Base58.DecodeData(invoiceId), string.Join(" ", terms.Where(t => !String.IsNullOrWhiteSpace(t)))); tx.Commit(); } }); } public async Task UpdateInvoiceStatus(string invoiceId, InvoiceState invoiceState) { using (var context = _ContextFactory.CreateContext()) { var invoiceData = await context.FindAsync(invoiceId).ConfigureAwait(false); if (invoiceData == null) return; invoiceData.Status = InvoiceState.ToString(invoiceState.Status); invoiceData.ExceptionStatus = InvoiceState.ToString(invoiceState.ExceptionStatus); await context.SaveChangesAsync().ConfigureAwait(false); } } public async Task UpdatePaidInvoiceToInvalid(string invoiceId) { using (var context = _ContextFactory.CreateContext()) { var invoiceData = await context.FindAsync(invoiceId).ConfigureAwait(false); if (invoiceData == null || !invoiceData.GetInvoiceState().CanMarkInvalid()) return; invoiceData.Status = "invalid"; invoiceData.ExceptionStatus = "marked"; await context.SaveChangesAsync().ConfigureAwait(false); } } public async Task UpdatePaidInvoiceToComplete(string invoiceId) { using (var context = _ContextFactory.CreateContext()) { var invoiceData = await context.FindAsync(invoiceId).ConfigureAwait(false); if (invoiceData == null || !invoiceData.GetInvoiceState().CanMarkComplete()) return; invoiceData.Status = "complete"; invoiceData.ExceptionStatus = "marked"; await context.SaveChangesAsync().ConfigureAwait(false); } } public async Task GetInvoice(string id, bool inludeAddressData = false) { using (var context = _ContextFactory.CreateContext()) { IQueryable query = context .Invoices .Include(o => o.Payments) .Include(o => o.RefundAddresses); if (inludeAddressData) query = query.Include(o => o.HistoricalAddressInvoices).Include(o => o.AddressInvoices); query = query.Where(i => i.Id == id); var invoice = await query.FirstOrDefaultAsync().ConfigureAwait(false); if (invoice == null) return null; return ToEntity(invoice); } } private InvoiceEntity ToEntity(Data.InvoiceData invoice) { var entity = ToObject(invoice.Blob, null); PaymentMethodDictionary paymentMethods = null; #pragma warning disable CS0618 entity.Payments = invoice.Payments.Select(p => { var paymentEntity = ToObject(p.Blob, null); paymentEntity.Accounted = p.Accounted; // PaymentEntity on version 0 does not have their own fee, because it was assumed that the payment method have fixed fee. // We want to hide this legacy detail in InvoiceRepository, so we fetch the fee from the PaymentMethod and assign it to the PaymentEntity. if (paymentEntity.Version == 0) { if (paymentMethods == null) paymentMethods = entity.GetPaymentMethods(null); var paymentMethodDetails = paymentMethods.TryGet(paymentEntity.GetPaymentMethodId())?.GetPaymentMethodDetails(); if (paymentMethodDetails != null) // == null should never happen, but we never know. paymentEntity.NetworkFee = paymentMethodDetails.GetNextNetworkFee(); } return paymentEntity; }) .OrderBy(a => a.ReceivedTime).ToList(); #pragma warning restore CS0618 var state = invoice.GetInvoiceState(); entity.ExceptionStatus = state.ExceptionStatus; entity.Status = state.Status; entity.RefundMail = invoice.CustomerEmail; entity.Refundable = invoice.RefundAddresses.Count != 0; if (invoice.HistoricalAddressInvoices != null) { entity.HistoricalAddresses = invoice.HistoricalAddressInvoices.ToArray(); } if (invoice.AddressInvoices != null) { entity.AvailableAddressHashes = invoice.AddressInvoices.Select(a => a.GetAddress() + a.GetpaymentMethodId().ToString()).ToHashSet(); } if (invoice.Events != null) { entity.Events = invoice.Events.OrderBy(c => c.Timestamp).ToList(); } return entity; } private IQueryable GetInvoiceQuery(ApplicationDbContext context, InvoiceQuery queryObject) { IQueryable query = context.Invoices; if (!string.IsNullOrEmpty(queryObject.InvoiceId)) { query = query.Where(i => i.Id == queryObject.InvoiceId); } if (queryObject.StoreId != null && queryObject.StoreId.Length > 0) { var stores = queryObject.StoreId.ToHashSet(); query = query.Where(i => stores.Contains(i.StoreDataId)); } if (queryObject.UserId != null) { query = query.Where(i => i.StoreData.UserStores.Any(u => u.ApplicationUserId == queryObject.UserId)); } if (!string.IsNullOrEmpty(queryObject.TextSearch)) { var ids = new HashSet(SearchInvoice(queryObject.TextSearch)); if (ids.Count == 0) { // Hacky way to return an empty query object. The nice way is much too elaborate: // https://stackoverflow.com/questions/33305495/how-to-return-empty-iqueryable-in-an-async-repository-method return query.Where(x => false); } query = query.Where(i => ids.Contains(i.Id)); } if (queryObject.StartDate != null) query = query.Where(i => queryObject.StartDate.Value <= i.Created); if (queryObject.EndDate != null) query = query.Where(i => i.Created <= queryObject.EndDate.Value); if (queryObject.OrderId != null && queryObject.OrderId.Length > 0) { var statusSet = queryObject.OrderId.ToHashSet(); query = query.Where(i => statusSet.Contains(i.OrderId)); } if (queryObject.ItemCode != null && queryObject.ItemCode.Length > 0) { var statusSet = queryObject.ItemCode.ToHashSet(); query = query.Where(i => statusSet.Contains(i.ItemCode)); } if (queryObject.Status != null && queryObject.Status.Length > 0) { var statusSet = queryObject.Status.ToHashSet(); query = query.Where(i => statusSet.Contains(i.Status)); } if (queryObject.Unusual != null) { var unused = queryObject.Unusual.Value; query = query.Where(i => unused == (i.Status == "invalid" || i.ExceptionStatus != null)); } if (queryObject.ExceptionStatus != null && queryObject.ExceptionStatus.Length > 0) { var exceptionStatusSet = queryObject.ExceptionStatus.Select(s => NormalizeExceptionStatus(s)).ToHashSet(); query = query.Where(i => exceptionStatusSet.Contains(i.ExceptionStatus)); } query = query.OrderByDescending(q => q.Created); if (queryObject.Skip != null) query = query.Skip(queryObject.Skip.Value); if (queryObject.Count != null) query = query.Take(queryObject.Count.Value); return query; } public async Task GetInvoicesTotal(InvoiceQuery queryObject) { using (var context = _ContextFactory.CreateContext()) { var query = GetInvoiceQuery(context, queryObject); return await query.CountAsync(); } } public async Task GetInvoices(InvoiceQuery queryObject) { using (var context = _ContextFactory.CreateContext()) { var query = GetInvoiceQuery(context, queryObject); query = query.Include(o => o.Payments) .Include(o => o.RefundAddresses); if (queryObject.IncludeAddresses) query = query.Include(o => o.HistoricalAddressInvoices).Include(o => o.AddressInvoices); if (queryObject.IncludeEvents) query = query.Include(o => o.Events); var data = await query.ToArrayAsync().ConfigureAwait(false); return data.Select(ToEntity).ToArray(); } } private string NormalizeExceptionStatus(string status) { status = status.ToLowerInvariant(); switch (status) { case "paidover": case "over": case "overpaid": status = "paidOver"; break; case "paidlate": case "late": status = "paidLate"; break; case "paidpartial": case "underpaid": case "partial": status = "paidPartial"; break; } return status; } public async Task AddRefundsAsync(string invoiceId, TxOut[] outputs, Network network) { if (outputs.Length == 0) return; outputs = outputs.Take(10).ToArray(); using (var context = _ContextFactory.CreateContext()) { int i = 0; foreach (var output in outputs) { context.RefundAddresses.Add(new RefundAddressesData() { Id = invoiceId + "-" + i, InvoiceDataId = invoiceId, Blob = ToBytes(output, network) }); i++; } await context.SaveChangesAsync().ConfigureAwait(false); } var addresses = outputs.Select(o => o.ScriptPubKey.GetDestinationAddress(network)).Where(a => a != null).ToArray(); AddToTextSearch(invoiceId, addresses.Select(a => a.ToString()).ToArray()); } /// /// Add a payment to an invoice /// /// /// /// /// /// /// The PaymentEntity or null if already added public async Task AddPayment(string invoiceId, DateTimeOffset date, CryptoPaymentData paymentData, BTCPayNetwork network, bool accounted = false) { using (var context = _ContextFactory.CreateContext()) { var invoice = context.Invoices.Find(invoiceId); if (invoice == null) return null; InvoiceEntity invoiceEntity = ToObject(invoice.Blob, network.NBitcoinNetwork); PaymentMethod paymentMethod = invoiceEntity.GetPaymentMethod(new PaymentMethodId(network.CryptoCode, paymentData.GetPaymentType()), null); IPaymentMethodDetails paymentMethodDetails = paymentMethod.GetPaymentMethodDetails(); PaymentEntity entity = new PaymentEntity { Version = 1, #pragma warning disable CS0618 CryptoCode = network.CryptoCode, #pragma warning restore CS0618 ReceivedTime = date.UtcDateTime, Accounted = accounted, NetworkFee = paymentMethodDetails.GetNextNetworkFee() }; entity.SetCryptoPaymentData(paymentData); if (paymentMethodDetails is Payments.Bitcoin.BitcoinLikeOnChainPaymentMethod bitcoinPaymentMethod && bitcoinPaymentMethod.NetworkFeeMode == NetworkFeeMode.MultiplePaymentsOnly && bitcoinPaymentMethod.NextNetworkFee == Money.Zero) { bitcoinPaymentMethod.NextNetworkFee = bitcoinPaymentMethod.FeeRate.GetFee(100); // assume price for 100 bytes paymentMethod.SetPaymentMethodDetails(bitcoinPaymentMethod); invoiceEntity.SetPaymentMethod(paymentMethod); invoice.Blob = ToBytes(invoiceEntity, network.NBitcoinNetwork); } PaymentData data = new PaymentData { Id = paymentData.GetPaymentId(), Blob = ToBytes(entity, null), InvoiceDataId = invoiceId, Accounted = accounted }; context.Payments.Add(data); try { await context.SaveChangesAsync().ConfigureAwait(false); } catch (DbUpdateException) { return null; } // Already exists AddToTextSearch(invoiceId, paymentData.GetSearchTerms()); return entity; } } public async Task UpdatePayments(List payments) { if (payments.Count == 0) return; using (var context = _ContextFactory.CreateContext()) { foreach (var payment in payments) { var paymentData = payment.GetCryptoPaymentData(); var data = new PaymentData(); data.Id = paymentData.GetPaymentId(); data.Accounted = payment.Accounted; data.Blob = ToBytes(payment, null); context.Attach(data); context.Entry(data).Property(o => o.Accounted).IsModified = true; context.Entry(data).Property(o => o.Blob).IsModified = true; } await context.SaveChangesAsync().ConfigureAwait(false); } } private T ToObject(byte[] value, Network network) { return NBitcoin.JsonConverters.Serializer.ToObject(ZipUtils.Unzip(value), network); } private byte[] ToBytes(T obj, Network network) { return ZipUtils.Zip(NBitcoin.JsonConverters.Serializer.ToString(obj, network)); } private T Clone(T invoice, Network network) { return NBitcoin.JsonConverters.Serializer.ToObject(ToString(invoice, network), network); } private string ToString(T data, Network network) { return NBitcoin.JsonConverters.Serializer.ToString(data, network); } public void Dispose() { if (_Engine != null) _Engine.Dispose(); if (_IndexerThread != null) _IndexerThread.Dispose(); } } public class InvoiceQuery { public string[] StoreId { get; set; } public string UserId { get; set; } public string TextSearch { get; set; } public DateTimeOffset? StartDate { get; set; } public DateTimeOffset? EndDate { get; set; } public int? Skip { get; set; } public int? Count { get; set; } public string[] OrderId { get; set; } public string[] ItemCode { get; set; } public bool? Unusual { get; set; } public string[] Status { get; set; } public string[] ExceptionStatus { get; set; } public string InvoiceId { get; set; } public bool IncludeAddresses { get; set; } public bool IncludeEvents { get; set; } } }