From bed47de5e3882791b7927ed8be0fe1c3e15c1a3a Mon Sep 17 00:00:00 2001 From: Fabrice Drouin Date: Fri, 19 Apr 2019 22:35:12 +0200 Subject: [PATCH] Live channel database backup (#951) * Backup running channel database when needed Every time our channel database needs to be persisted, we create a backup which is always safe to copy even when the system is busy. * Upgrade sqlite-jdbc to 3.27.2.1 * BackupHandler: use a specific bounded mailbox BackupHandler is now private, users have to call BackupHandler.props() which always specifies our custom bounded maibox. * BackupHandler: use a specific threadpool with a single thread * Add backup notification script Once a new backup has been created, call an optional user defined script. --- README.md | 20 ++++++ eclair-core/pom.xml | 2 +- eclair-core/src/main/resources/reference.conf | 16 +++++ .../main/scala/fr/acinq/eclair/Setup.scala | 14 +++-- .../fr/acinq/eclair/db/BackupHandler.scala | 62 +++++++++++++++++++ .../scala/fr/acinq/eclair/db/Databases.scala | 9 ++- .../acinq/eclair/db/sqlite/SqliteUtils.scala | 2 +- .../acinq/eclair/db/BackupHandlerSpec.scala | 38 ++++++++++++ 8 files changed, 156 insertions(+), 7 deletions(-) create mode 100644 eclair-core/src/main/scala/fr/acinq/eclair/db/BackupHandler.scala create mode 100644 eclair-core/src/test/scala/fr/acinq/eclair/db/BackupHandlerSpec.scala diff --git a/README.md b/README.md index d2073c831..ad19f95e4 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,26 @@ Eclair uses [`logback`](https://logback.qos.ch) for logging. To use a different java -Dlogback.configurationFile=/path/to/logback-custom.xml -jar eclair-node-gui--.jar ``` +#### Backup + +The files that you need to backup are located in your data directory. You must backup: +- your seed (`seed.dat`) +- your channel database (`eclair.bak` under directory `mainnet`, `testnet` or `regtest` depending on which chain you're running on) + +Your seed never changes once it has been created, but your channels will change whenever you receive or send payments. Eclair will +create and maintain a snapshot of its database, named `eclair.bak`, in your data directory, and update it when needed. This file is +always consistent and safe to use even when Eclair is running, and this is what you should backup regularly. + +For example you could configure a `cron` task for your backup job. Or you could configure an optional notification script to be called by eclair once a new database snapshot has been created, using the following option: +``` +eclair.backup-notify-script = "absolute-path-to-your-script" +``` +Make sure that your script is executable and uses an absolute path name for `eclair.bak`. + +Note that depending on your filesystem, in your backup process we recommend first moving `eclair.bak` to some temporary file +before copying that file to your final backup location. + + ## Docker A [Dockerfile](Dockerfile) image is built on each commit on [docker hub](https://hub.docker.com/r/acinq/eclair) for running a dockerized eclair-node. diff --git a/eclair-core/pom.xml b/eclair-core/pom.xml index e2541f1a0..91a5c4be9 100644 --- a/eclair-core/pom.xml +++ b/eclair-core/pom.xml @@ -203,7 +203,7 @@ org.xerial sqlite-jdbc - 3.21.0.1 + 3.27.2.1 diff --git a/eclair-core/src/main/resources/reference.conf b/eclair-core/src/main/resources/reference.conf index a527b12f9..f56a78f6f 100644 --- a/eclair-core/src/main/resources/reference.conf +++ b/eclair-core/src/main/resources/reference.conf @@ -16,6 +16,9 @@ eclair { use-old-api = false } + // override this with a script/exe that will be called everytime a new database backup has been created + backup-notify-script = "" + watcher-type = "bitcoind" // other *experimental* values include "electrum" bitcoind { @@ -134,3 +137,16 @@ eclair { private-key-file = "tor.dat" } } + +// do not edit or move this section +eclair { + backup-mailbox { + mailbox-type = "akka.dispatch.BoundedMailbox" + mailbox-capacity = 1 + mailbox-push-timeout-time = 0 + } + backup-dispatcher { + executor = "thread-pool-executor" + type = PinnedDispatcher + } +} \ No newline at end of file diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala b/eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala index 7f9df23da..3884a4640 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala @@ -43,7 +43,7 @@ import fr.acinq.eclair.blockchain.fee.{ConstantFeeProvider, _} import fr.acinq.eclair.blockchain.{EclairWallet, _} import fr.acinq.eclair.channel.Register import fr.acinq.eclair.crypto.LocalKeyManager -import fr.acinq.eclair.db.Databases +import fr.acinq.eclair.db.{BackupHandler, Databases} import fr.acinq.eclair.io.{Authenticator, Server, Switchboard} import fr.acinq.eclair.payment._ import fr.acinq.eclair.router._ @@ -88,11 +88,12 @@ class Setup(datadir: File, val config = NodeParams.loadConfiguration(datadir, overrideDefaults) val seed = seed_opt.getOrElse(NodeParams.getSeed(datadir)) val chain = config.getString("chain") + val chaindir = new File(datadir, chain) val keyManager = new LocalKeyManager(seed, NodeParams.makeChainHash(chain)) val database = db match { case Some(d) => d - case None => Databases.sqliteJDBC(new File(datadir, chain)) + case None => Databases.sqliteJDBC(chaindir) } val nodeParams = NodeParams.makeNodeParams(config, keyManager, initTor(), database) @@ -223,8 +224,6 @@ class Setup(datadir: File, wallet = bitcoin match { case Bitcoind(bitcoinClient) => new BitcoinCoreWallet(bitcoinClient) case Electrum(electrumClient) => - // TODO: DRY - val chaindir = new File(datadir, chain) val sqlite = DriverManager.getConnection(s"jdbc:sqlite:${new File(chaindir, "wallet.sqlite")}") val walletDb = new SqliteWalletDb(sqlite) val electrumWallet = system.actorOf(ElectrumWallet.props(seed, electrumClient, ElectrumWallet.WalletParameters(nodeParams.chainHash, walletDb)), "electrum-wallet") @@ -234,7 +233,14 @@ class Setup(datadir: File, _ = wallet.getFinalAddress.map { case address => logger.info(s"initial wallet address=$address") } + // do not change the name of this actor. it is used in the configuration to specify a custom bounded mailbox + backupHandler = system.actorOf(SimpleSupervisor.props( + BackupHandler.props( + nodeParams.db, + new File(chaindir, "eclair.bak"), + if (config.hasPath("backup-notify-script")) Some(config.getString("backup-notify-script")) else None + ),"backuphandler", SupervisorStrategy.Resume)) audit = system.actorOf(SimpleSupervisor.props(Auditor.props(nodeParams), "auditor", SupervisorStrategy.Resume)) paymentHandler = system.actorOf(SimpleSupervisor.props(config.getString("payment-handler") match { case "local" => LocalPaymentHandler.props(nodeParams) diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/db/BackupHandler.scala b/eclair-core/src/main/scala/fr/acinq/eclair/db/BackupHandler.scala new file mode 100644 index 000000000..c607c268a --- /dev/null +++ b/eclair-core/src/main/scala/fr/acinq/eclair/db/BackupHandler.scala @@ -0,0 +1,62 @@ +package fr.acinq.eclair.db + +import java.io.File + +import akka.actor.{Actor, ActorLogging, Props} +import akka.dispatch.{BoundedMessageQueueSemantics, RequiresMessageQueue} +import fr.acinq.eclair.channel.ChannelPersisted + +import scala.sys.process.Process +import scala.util.{Failure, Success, Try} + + +/** + * This actor will synchronously make a backup of the database it was initialized with whenever it receives + * a ChannelPersisted event. + * To avoid piling up messages and entering an endless backup loop, it is supposed to be used with a bounded mailbox + * with a single item: + * + * backup-mailbox { + * mailbox-type = "akka.dispatch.BoundedMailbox" + * mailbox-capacity = 1 + * mailbox-push-timeout-time = 0 + * } + * + * Messages that cannot be processed will be sent to dead letters + * + * @param databases database to backup + * @param backupFile backup file + * + * Constructor is private so users will have to use BackupHandler.props() which always specific a custom mailbox + */ +class BackupHandler private(databases: Databases, backupFile: File, backupScript_opt: Option[String]) extends Actor with RequiresMessageQueue[BoundedMessageQueueSemantics] with ActorLogging { + + // we listen to ChannelPersisted events, which will trigger a backup + context.system.eventStream.subscribe(self, classOf[ChannelPersisted]) + + def receive = { + case persisted: ChannelPersisted => + val start = System.currentTimeMillis() + val tmpFile = new File(backupFile.getAbsolutePath.concat(".tmp")) + databases.backup(tmpFile) + val result = tmpFile.renameTo(backupFile) + require(result, s"cannot rename $tmpFile to $backupFile") + val end = System.currentTimeMillis() + log.info(s"database backup triggered by channelId=${persisted.channelId} took ${end - start}ms") + backupScript_opt.foreach(backupScript => { + Try { + // run the script in the current thread and wait until it terminates + Process(backupScript).! + } match { + case Success(exitCode) => log.info(s"backup notify script $backupScript returned $exitCode") + case Failure(cause) => log.warning(s"cannot start backup notify script $backupScript: $cause") + } + }) + } +} + +object BackupHandler { + // using this method is the only way to create a BackupHandler actor + // we make sure that it uses a custom bounded mailbox, and a custom pinned dispatcher (i.e our actor will have its own thread pool with 1 single thread) + def props(databases: Databases, backupFile: File, backupScript_opt: Option[String]) = Props(new BackupHandler(databases, backupFile, backupScript_opt)).withMailbox("eclair.backup-mailbox").withDispatcher("eclair.backup-dispatcher") +} diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/db/Databases.scala b/eclair-core/src/main/scala/fr/acinq/eclair/db/Databases.scala index ff2badd0a..451ab41e4 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/db/Databases.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/db/Databases.scala @@ -19,6 +19,7 @@ trait Databases { val pendingRelay: PendingRelayDb + def backup(file: File) : Unit } object Databases { @@ -45,6 +46,12 @@ object Databases { override val peers = new SqlitePeersDb(eclairJdbc) override val payments = new SqlitePaymentsDb(eclairJdbc) override val pendingRelay = new SqlitePendingRelayDb(eclairJdbc) + override def backup(file: File): Unit = { + SqliteUtils.using(eclairJdbc.createStatement()) { + statement => { + statement.executeUpdate(s"backup to ${file.getAbsolutePath}") + } + } + } } - } \ No newline at end of file diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqliteUtils.scala b/eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqliteUtils.scala index e14611243..5d79c8df3 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqliteUtils.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqliteUtils.scala @@ -109,7 +109,7 @@ object SqliteUtils { * Obtain an exclusive lock on a sqlite database. This is useful when we want to make sure that only one process * accesses the database file (see https://www.sqlite.org/pragma.html). * - * The lock will be kept until the database is closed, or if the locking mode is explicitely reset. + * The lock will be kept until the database is closed, or if the locking mode is explicitly reset. * * @param sqlite */ diff --git a/eclair-core/src/test/scala/fr/acinq/eclair/db/BackupHandlerSpec.scala b/eclair-core/src/test/scala/fr/acinq/eclair/db/BackupHandlerSpec.scala new file mode 100644 index 000000000..d57d9a072 --- /dev/null +++ b/eclair-core/src/test/scala/fr/acinq/eclair/db/BackupHandlerSpec.scala @@ -0,0 +1,38 @@ +package fr.acinq.eclair.db + +import java.io.File +import java.sql.DriverManager +import java.util.UUID + +import akka.actor.{ActorSystem, Props} +import akka.testkit.TestKit +import fr.acinq.eclair.channel.ChannelPersisted +import fr.acinq.eclair.db.sqlite.SqliteChannelsDb +import fr.acinq.eclair.{TestConstants, TestUtils, randomBytes32} +import org.scalatest.FunSuiteLike + +import scala.concurrent.duration._ + +class BackupHandlerSpec extends TestKit(ActorSystem("test")) with FunSuiteLike { + + test("process backups") { + val db = TestConstants.inMemoryDb() + val wip = new File(TestUtils.BUILD_DIRECTORY, s"wip-${UUID.randomUUID()}") + val dest = new File(TestUtils.BUILD_DIRECTORY, s"backup-${UUID.randomUUID()}") + wip.deleteOnExit() + dest.deleteOnExit() + val channel = ChannelStateSpec.normal + db.channels.addOrUpdateChannel(channel) + assert(db.channels.listLocalChannels() == Seq(channel)) + + val handler = system.actorOf(BackupHandler.props(db, dest, None)) + handler ! ChannelPersisted(null, TestConstants.Alice.nodeParams.nodeId, randomBytes32, null) + handler ! ChannelPersisted(null, TestConstants.Alice.nodeParams.nodeId, randomBytes32, null) + handler ! ChannelPersisted(null, TestConstants.Alice.nodeParams.nodeId, randomBytes32, null) + awaitCond(dest.exists(), 5 seconds) + + val db1 = new SqliteChannelsDb(DriverManager.getConnection(s"jdbc:sqlite:$dest")) + val check = db1.listLocalChannels() + assert(check == Seq(channel)) + } +}