mirrors/btcpayserver
1
0
Fork 0
mirror of https://github.com/btcpayserver/btcpayserver.git synced 2024-11-19 01:43:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add ability to migrate from MySQL and SQLite with EF (#4614)

Browse Source
This commit is contained in:
Nicolas Dorier 2023-02-10 11:43:46 +09:00 committed by GitHub
parent 753ffd401b
commit 1f8bc5b490
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 335 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
BTCPayServer.Abstractions/Configuration/DataDirectories.cs
View File

@ -1,3 +1,5 @@
using System.IO;
namespace BTCPayServer.Configuration
{
public class DataDirectories
@ -7,5 +9,12 @@ namespace BTCPayServer.Configuration
public string TempStorageDir { get; set; }
public string StorageDir { get; set; }
public string TempDir { get; set; }
public string ToDatadirFullPath(string path)
{
if (Path.IsPathRooted(path))
return path;
return Path.Combine(DataDir, path);
}
}
}

6
BTCPayServer.Data/ApplicationDbContext.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using BTCPayServer.Data.Data;
using Microsoft.AspNetCore.Identity.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore;
@ -31,6 +32,11 @@ namespace BTCPayServer.Data
_designTime = designTime;
}
public async Task<string?> GetMigrationState()
{
return (await Settings.FromSqlRaw("SELECT \"Id\", \"Value\" FROM \"Settings\" WHERE \"Id\"='MigrationData'").AsNoTracking().FirstOrDefaultAsync())?.Value;
}
public DbSet<AddressInvoiceData> AddressInvoices { get; set; }
public DbSet<APIKeyData> ApiKeys { get; set; }
public DbSet<AppData> Apps { get; set; }

12
BTCPayServer/Configuration/BTCPayServerOptions.cs
View File

@ -65,10 +65,14 @@ namespace BTCPayServer.Configuration
if (conf.GetOrDefault<bool>("launchsettings", false) && NetworkType != ChainName.Regtest)
throw new ConfigException($"You need to run BTCPayServer with the run.sh or run.ps1 script");
if (conf.GetOrDefault<string>("SQLITEFILE", null) != null)
Logs.Configuration.LogWarning("SQLITE backend support is deprecated and will be soon out of support");
if (conf.GetOrDefault<string>("MYSQL", null) != null)
Logs.Configuration.LogWarning("MYSQL backend support is deprecated and will be soon out of support");
if (conf.GetOrDefault<string>("POSTGRES", null) == null)
{
if (conf.GetOrDefault<string>("SQLITEFILE", null) != null)
Logs.Configuration.LogWarning("SQLITE backend support is out of support. Please migrate to Postgres by following the following instructions https://github.com/btcpayserver/btcpayserver/blob/master/docs/db-migration.md");
if (conf.GetOrDefault<string>("MYSQL", null) != null)
Logs.Configuration.LogWarning("MYSQL backend support is out of support. Please migrate to Postgres by following the following instructions (https://github.com/btcpayserver/btcpayserver/blob/master/docs/db-migration.md)");
}
DockerDeployment = conf.GetOrDefault<bool>("dockerdeployment", true);
TorrcFile = conf.GetOrDefault<string>("torrcfile", null);
TorServices = conf.GetOrDefault<string>("torservices", null)

9
BTCPayServer/Hosting/BTCPayServerServices.cs
View File

@ -128,7 +128,10 @@ namespace BTCPayServer.Hosting
services.TryAddSingleton<BTCPayServerOptions>(o =>
o.GetRequiredService<IOptions<BTCPayServerOptions>>().Value);
// Don't move this StartupTask, we depend on it being right here
if (configuration["POSTGRES"] != null && (configuration["SQLITEFILE"] != null || configuration["MYSQL"] != null))
services.AddStartupTask<ToPostgresMigrationStartupTask>();
services.AddStartupTask<MigrationStartupTask>();
//
AddSettingsAccessor<PoliciesSettings>(services);
AddSettingsAccessor<ThemeSettings>(services);
@ -174,12 +177,8 @@ namespace BTCPayServer.Hosting
}
else if (!string.IsNullOrEmpty(sqliteFileName))
{
var connStr = "Data Source=" + (Path.IsPathRooted(sqliteFileName)
? sqliteFileName
: Path.Combine(datadirs.Value.DataDir, sqliteFileName));
options.DatabaseType = DatabaseType.Sqlite;
options.ConnectionString = connStr;
options.ConnectionString = "Data Source=" + datadirs.Value.ToDatadirFullPath(sqliteFileName);
}
else
{

9
BTCPayServer/Hosting/MigrationStartupTask.cs
View File

@ -686,9 +686,16 @@ WHERE cte.""Id""=p.""Id""
retry:
try
{
await _DBContextFactory.CreateContext().Database.MigrateAsync();
var db = _DBContextFactory.CreateContext();
await db.Database.MigrateAsync();
if (db.Database.IsNpgsql())
{
if (await db.GetMigrationState() == "pending")
throw new ConfigException("This database hasn't been completely migrated, please retry migration by setting the BTCPAY_SQLITEFILE or BTCPAY_MYSQL setting on top of BTCPAY_POSTGRES");
}
}
// Starting up
catch (ConfigException) { throw; }
catch when (!cts.Token.IsCancellationRequested)
{
try

254
BTCPayServer/Hosting/ToPostgresMigrationStartupTask.cs Normal file
View File

@ -0,0 +1,254 @@
#nullable enable
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Amazon.Runtime.Internal.Util;
using AngleSharp.Text;
using BTCPayServer.Abstractions.Contracts;
using BTCPayServer.Configuration;
using BTCPayServer.Data;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Identity;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Metadata.Conventions;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MySqlConnector;
using NBXplorer;
using Newtonsoft.Json.Linq;
using Npgsql;
namespace BTCPayServer.Hosting
{
static class TopologySort
{
public static IEnumerable<ITable> OrderByTopology(this IEnumerable<ITable> tables)
{
var comparer = Comparer<ITable>.Create((a, b) => a.Name.CompareTo(b.Name));
return OrderByTopology(
tables,
t =>
{
if (t.Name == "Invoices")
return t.ForeignKeyConstraints.Select(f => f.PrincipalTable.Name).Where(f => f != "Refunds");
else
return t.ForeignKeyConstraints.Select(f => f.PrincipalTable.Name);
},
t => t.Name,
t => t,
comparer);
}
public static IEnumerable<TValue> OrderByTopology<T, TDepend, TValue>(
this IEnumerable<T> values,
Func<T, IEnumerable<TDepend>> dependsOn,
Func<T, TDepend> getKey,
Func<T, TValue> getValue,
IComparer<T>? solveTies = null) where T : notnull
{
var v = values.ToList();
return TopologicalSort(v, dependsOn, getKey, getValue, solveTies);
}
static List<TValue> TopologicalSort<T, TDepend, TValue>(this IReadOnlyCollection<T> nodes,
Func<T, IEnumerable<TDepend>> dependsOn,
Func<T, TDepend> getKey,
Func<T, TValue> getValue,
IComparer<T>? solveTies = null) where T : notnull
{
if (nodes.Count == 0)
return new List<TValue>();
if (getKey == null)
throw new ArgumentNullException(nameof(getKey));
if (getValue == null)
throw new ArgumentNullException(nameof(getValue));
solveTies = solveTies ?? Comparer<T>.Default;
List<TValue> result = new List<TValue>(nodes.Count);
HashSet<TDepend> allKeys = new HashSet<TDepend>(nodes.Count);
var noDependencies = new SortedDictionary<T, HashSet<TDepend>>(solveTies);
foreach (var node in nodes)
allKeys.Add(getKey(node));
var dependenciesByValues = nodes.ToDictionary(node => node,
node => new HashSet<TDepend>(dependsOn(node).Where(n => allKeys.Contains(n))));
foreach (var e in dependenciesByValues.Where(x => x.Value.Count == 0))
{
noDependencies.Add(e.Key, e.Value);
}
if (noDependencies.Count == 0)
{
throw new InvalidOperationException("Impossible to topologically sort a cyclic graph");
}
while (noDependencies.Count > 0)
{
var nodep = noDependencies.First();
noDependencies.Remove(nodep.Key);
dependenciesByValues.Remove(nodep.Key);
var elemKey = getKey(nodep.Key);
result.Add(getValue(nodep.Key));
foreach (var selem in dependenciesByValues)
{
if (selem.Value.Remove(elemKey) && selem.Value.Count == 0)
noDependencies.Add(selem.Key, selem.Value);
}
}
if (dependenciesByValues.Count != 0)
{
throw new InvalidOperationException("Impossible to topologically sort a cyclic graph");
}
return result;
}
}
public class ToPostgresMigrationStartupTask : IStartupTask
{
public ToPostgresMigrationStartupTask(
IConfiguration configuration,
IOptions<DataDirectories> datadirs,
ILogger<ToPostgresMigrationStartupTask> logger,
IWebHostEnvironment environment,
ApplicationDbContextFactory dbContextFactory)
{
Configuration = configuration;
Datadirs = datadirs;
Logger = logger;
Environment = environment;
DbContextFactory = dbContextFactory;
}
public IConfiguration Configuration { get; }
public IOptions<DataDirectories> Datadirs { get; }
public ILogger<ToPostgresMigrationStartupTask> Logger { get; }
public IWebHostEnvironment Environment { get; }
public ApplicationDbContextFactory DbContextFactory { get; }
public bool HasError { get; private set; }
public async Task ExecuteAsync(CancellationToken cancellationToken = default)
{
var p = Configuration.GetOrDefault<string?>("POSTGRES", null);
var sqlite = Configuration.GetOrDefault<string?>("SQLITEFILE", null);
var mysql = Configuration.GetOrDefault<string?>("MYSQL", null);
string migratingFrom;
ApplicationDbContext otherContext;
if (string.IsNullOrEmpty(p))
{
return;
}
else if (!string.IsNullOrEmpty(sqlite))
{
migratingFrom = "SQLite";
sqlite = Datadirs.Value.ToDatadirFullPath(sqlite);
if (!File.Exists(sqlite))
return;
otherContext = new ApplicationDbContext(new DbContextOptionsBuilder<ApplicationDbContext>().UseSqlite("Data Source=" + sqlite, o => o.CommandTimeout(60 * 60 * 10)).Options);
}
else if (!string.IsNullOrEmpty(mysql))
{
migratingFrom = "MySQL";
otherContext = new ApplicationDbContext(new DbContextOptionsBuilder<ApplicationDbContext>().UseMySql(mysql, ServerVersion.AutoDetect(mysql), o => o.CommandTimeout(60 * 60 * 10)).Options);
try
{
await otherContext.Settings.FirstOrDefaultAsync();
}
catch (MySqlException ex) when (ex.SqlState == "42000") // DB doesn't exists
{
return;
}
}
else
{
return;
}
if (await otherContext.Settings.FirstOrDefaultAsync() == null)
return;
{
var postgres = new NpgsqlConnectionStringBuilder(p);
using var postgresContext = new ApplicationDbContext(new DbContextOptionsBuilder<ApplicationDbContext>().UseNpgsql(p, o => o.CommandTimeout(60 * 60 * 10)).Options);
string? state;
try
{
state = await GetMigrationState(postgresContext);
if (state == "complete")
return;
if (state == null)
throw new ConfigException("This postgres database isn't created during a migration. Please use an empty database for postgres when migrating. If it's not a migration, remove --sqlitefile or --mysql settings.");
}
catch (NpgsqlException ex) when (ex.SqlState == PostgresErrorCodes.InvalidCatalogName) // DB doesn't exists
{
await postgresContext.Database.MigrateAsync();
state = "pending";
await SetMigrationState(postgresContext, migratingFrom, "pending");
}
Logger.LogInformation($"Migrating from {migratingFrom} to Postgres...");
if (state == "pending")
{
Logger.LogInformation($"There is a unfinished migration in postgres... dropping all tables");
foreach (var t in postgresContext.Model.GetRelationalModel().Tables.OrderByTopology())
{
await postgresContext.Database.ExecuteSqlRawAsync($"DROP TABLE IF EXISTS \"{t.Name}\" CASCADE");
}
await postgresContext.Database.ExecuteSqlRawAsync($"DROP TABLE IF EXISTS \"__EFMigrationsHistory\" CASCADE");
await postgresContext.Database.MigrateAsync();
}
else
{
throw new ConfigException("This database isn't created during a migration. Please use an empty database for postgres when migrating.");
}
await otherContext.Database.MigrateAsync();
await SetMigrationState(postgresContext, migratingFrom, "pending");
foreach (var t in postgresContext.Model.GetRelationalModel().Tables.OrderByTopology())
{
var typeMapping = t.EntityTypeMappings.Single();
var query = (IQueryable<object>)otherContext.GetType().GetMethod("Set", new Type[0])!.MakeGenericMethod(typeMapping.EntityType.ClrType).Invoke(otherContext, null)!;
Logger.LogInformation($"Migrating table: " + t.Name);
var rows = await query.ToListAsync();
foreach (var row in rows)
{
// There is as circular deps between invoice and refund.
if (row is InvoiceData id)
id.CurrentRefundId = null;
postgresContext.Entry(row).State = EntityState.Added;
}
await postgresContext.SaveChangesAsync();
postgresContext.ChangeTracker.Clear();
}
foreach (var invoice in otherContext.Invoices.AsNoTracking().Where(i => i.CurrentRefundId != null))
{
postgresContext.Entry(invoice).State = EntityState.Modified;
}
await postgresContext.SaveChangesAsync();
postgresContext.ChangeTracker.Clear();
await SetMigrationState(postgresContext, migratingFrom, "complete");
}
otherContext.Dispose();
SqliteConnection.ClearAllPools();
MySqlConnection.ClearAllPools();
Logger.LogInformation($"Migration to postgres from {migratingFrom} successful");
}
private static async Task<string?> GetMigrationState(ApplicationDbContext postgresContext)
{
return (await postgresContext.Settings.FromSqlRaw("SELECT \"Id\", \"Value\" FROM \"Settings\" WHERE \"Id\"='MigrationData'").AsNoTracking().FirstOrDefaultAsync())?.Value;
}
private static async Task SetMigrationState(ApplicationDbContext postgresContext, string migratingFrom, string state)
{
await postgresContext.Database.ExecuteSqlRawAsync(
"INSERT INTO \"Settings\" VALUES ('MigrationData', @p0::JSONB) ON CONFLICT (\"Id\") DO UPDATE SET \"Value\"=@p0::JSONB",
new[] { $"{{ \"from\": \"{migratingFrom}\", \"state\": \"{state}\" }}" });
}
}
}

46
docs/db-migration.md Normal file
View File

@ -0,0 +1,46 @@
# Migration from SQLite and MySQL to Postgres
## Introduction
This document is intended for BTCPay Server integrators such as Raspiblitz, Umbrel, Embassy OS or anybody running BTCPay Server on SQLite or MySql.
If you are a user of an integrated solution, please contact the integrator directly and provide them with the link to this document.
BTCPay Server has for long time supported three different backends:
1. Postgres
2. SQLite
3. MySql
While most of our users are using the Postgres backend, maintaining supports for all those databases has been very challenging, and Postgres is the only one part of our test suite.
As a result, we regret to inform you that we decided to stop the support of MySql and SQLite.
We understand that dropping support might be painful for users and integrators of our product, and we will do our best to provide a migration path.
Please keep us informed if you experience any issues while migrating on [our community chat](https://chat.btcpayserver.org).
## Procedure
In order to successfully migrate, you will need to run BTCPay Server `1.7.8 or older`.
As a reminder there are three settings controlling the choice of backend of BTCPay Server which can be controller by command line, environment variable or configuration settings.
| Command line argument | Environment variable |
|---|---|
| --postgres | BTCPAY_POSTGRES="..." |
| --mysql | BTCPAY_MYSQL="..." |
| --sqlitefile | BTCPAY_SQLITEFILE="blah.db" |
If you are currently using `mysql` or `sqlitefile`, and you wish to migrate to postgres, you simply need to add the command line argument `--postgres` or the environment variable `BTCPAY_POSTGRES` pointing to a fresh postgres database.
From `1.7.8`, BTCPay Server will interprete this and attempt to copy the data from mysql and sqlite into the new postgres database.
Note that once the migration is complete, the old `mysql` and `sqlite` settings will simply be ignored.
If the migration fails, you can revert the `postgres` setting you added, so the next restart will run on the old unsupported database. You can retry a migration by adding the `postgres` setting again.
## Known issues
* The migration script isn't very optimized, and will attempt to load every table in memory. If your `sqlite` or `mysql` database is too big, you may experience an Out Of Memory issue. If that happen to you, please contact us.
* There are no migration for plugin's data.