Live channel database backup (#951)

* Backup running channel database when needed

Every time our channel database needs to be persisted, we create a backup which is always
safe to copy even when the system is busy.

* Upgrade sqlite-jdbc to 3.27.2.1

* BackupHandler: use a specific bounded mailbox

BackupHandler is now private, users have to call BackupHandler.props() which always
specifies our custom bounded maibox.

* BackupHandler: use a specific threadpool with a single thread

* Add backup notification script

Once a new backup has been created, call an optional user defined script.

README.md

@@ -136,6 +136,26 @@ Eclair uses [`logback`](https://logback.qos.ch) for logging. To use a different
 java -Dlogback.configurationFile=/path/to/logback-custom.xml -jar eclair-node-gui-<version>-<commit_id>.jar
 ```
 
+#### Backup
+
+The files that you need to backup are located in your data directory. You must backup:
+- your seed (`seed.dat`)
+- your channel database (`eclair.bak` under directory `mainnet`, `testnet` or `regtest` depending on which chain you're running on)
+
+Your seed never changes once it has been created, but your channels will change whenever you receive or send payments. Eclair will
+create and maintain a snapshot of its database, named `eclair.bak`, in your data directory, and update it when needed. This file is 
+always consistent and safe to use even when Eclair is running, and this is what you should backup regularly.
+
+For example you could configure a `cron` task for your backup job. Or you could configure an optional notification script to be called by eclair once a new database snapshot has been created, using the following option:
+```
+eclair.backup-notify-script = "absolute-path-to-your-script"
+```
+Make sure that your script is executable and uses an absolute path name for `eclair.bak`.
+
+Note that depending on your filesystem, in your backup process we recommend first moving `eclair.bak` to some temporary file 
+before copying that file to your final backup location.
+
+
 ## Docker
 
 A [Dockerfile](Dockerfile) image is built on each commit on [docker hub](https://hub.docker.com/r/acinq/eclair) for running a dockerized eclair-node.

eclair-core/pom.xml

@@ -203,7 +203,7 @@
         <dependency>
             <groupId>org.xerial</groupId>
             <artifactId>sqlite-jdbc</artifactId>
-            <version>3.21.0.1</version>
+            <version>3.27.2.1</version>
         </dependency>
         <dependency>
             <!-- This is to get rid of '[WARNING] warning: Class javax.annotation.Nonnull not found - continuing with a stub.' compile errors -->

eclair-core/src/main/resources/reference.conf

@@ -16,6 +16,9 @@ eclair {
     use-old-api = false
   }
 
+  // override this with a script/exe that will be called everytime a new database backup has been created
+  backup-notify-script = ""
+
   watcher-type = "bitcoind" // other *experimental* values include "electrum"
 
   bitcoind {
@@ -134,3 +137,16 @@ eclair {
     private-key-file = "tor.dat"
   }
 }
+
+// do not edit or move this section
+eclair {
+  backup-mailbox {
+    mailbox-type = "akka.dispatch.BoundedMailbox"
+    mailbox-capacity = 1
+    mailbox-push-timeout-time = 0
+  }
+  backup-dispatcher {
+    executor = "thread-pool-executor"
+    type = PinnedDispatcher
+  }
+}

eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala

@@ -43,7 +43,7 @@ import fr.acinq.eclair.blockchain.fee.{ConstantFeeProvider, _}
 import fr.acinq.eclair.blockchain.{EclairWallet, _}
 import fr.acinq.eclair.channel.Register
 import fr.acinq.eclair.crypto.LocalKeyManager
-import fr.acinq.eclair.db.Databases
+import fr.acinq.eclair.db.{BackupHandler, Databases}
 import fr.acinq.eclair.io.{Authenticator, Server, Switchboard}
 import fr.acinq.eclair.payment._
 import fr.acinq.eclair.router._
@@ -88,11 +88,12 @@ class Setup(datadir: File,
   val config = NodeParams.loadConfiguration(datadir, overrideDefaults)
   val seed = seed_opt.getOrElse(NodeParams.getSeed(datadir))
   val chain = config.getString("chain")
+  val chaindir = new File(datadir, chain)
   val keyManager = new LocalKeyManager(seed, NodeParams.makeChainHash(chain))
 
   val database = db match {
     case Some(d) => d
-    case None => Databases.sqliteJDBC(new File(datadir, chain))
+    case None => Databases.sqliteJDBC(chaindir)
   }
 
   val nodeParams = NodeParams.makeNodeParams(config, keyManager, initTor(), database)
@@ -223,8 +224,6 @@ class Setup(datadir: File,
       wallet = bitcoin match {
         case Bitcoind(bitcoinClient) => new BitcoinCoreWallet(bitcoinClient)
         case Electrum(electrumClient) =>
-          // TODO: DRY
-          val chaindir = new File(datadir, chain)
           val sqlite = DriverManager.getConnection(s"jdbc:sqlite:${new File(chaindir, "wallet.sqlite")}")
           val walletDb = new SqliteWalletDb(sqlite)
           val electrumWallet = system.actorOf(ElectrumWallet.props(seed, electrumClient, ElectrumWallet.WalletParameters(nodeParams.chainHash, walletDb)), "electrum-wallet")
@@ -234,7 +233,14 @@ class Setup(datadir: File,
       _ = wallet.getFinalAddress.map {
         case address => logger.info(s"initial wallet address=$address")
       }
+      // do not change the name of this actor. it is used in the configuration to specify a custom bounded mailbox
 
+      backupHandler = system.actorOf(SimpleSupervisor.props(
+        BackupHandler.props(
+          nodeParams.db,
+          new File(chaindir, "eclair.bak"),
+          if (config.hasPath("backup-notify-script")) Some(config.getString("backup-notify-script")) else None
+        ),"backuphandler", SupervisorStrategy.Resume))
       audit = system.actorOf(SimpleSupervisor.props(Auditor.props(nodeParams), "auditor", SupervisorStrategy.Resume))
       paymentHandler = system.actorOf(SimpleSupervisor.props(config.getString("payment-handler") match {
         case "local" => LocalPaymentHandler.props(nodeParams)

eclair-core/src/main/scala/fr/acinq/eclair/db/BackupHandler.scala

@@ -0,0 +1,62 @@
+package fr.acinq.eclair.db
+
+import java.io.File
+
+import akka.actor.{Actor, ActorLogging, Props}
+import akka.dispatch.{BoundedMessageQueueSemantics, RequiresMessageQueue}
+import fr.acinq.eclair.channel.ChannelPersisted
+
+import scala.sys.process.Process
+import scala.util.{Failure, Success, Try}
+
+
+/**
+  * This actor will synchronously make a backup of the database it was initialized with whenever it receives
+  * a ChannelPersisted event.
+  * To avoid piling up messages and entering an endless backup loop, it is supposed to be used with a bounded mailbox
+  * with a single item:
+  *
+  * backup-mailbox {
+  *   mailbox-type = "akka.dispatch.BoundedMailbox"
+  *   mailbox-capacity = 1
+  *   mailbox-push-timeout-time = 0
+  * }
+  *
+  * Messages that cannot be processed will be sent to dead letters
+  *
+  * @param databases  database to backup
+  * @param backupFile backup file
+  *
+  * Constructor is private so users will have to use BackupHandler.props() which always specific a custom mailbox
+  */
+class BackupHandler private(databases: Databases, backupFile: File, backupScript_opt: Option[String]) extends Actor with RequiresMessageQueue[BoundedMessageQueueSemantics] with ActorLogging {
+
+  // we listen to ChannelPersisted events, which will trigger a backup
+  context.system.eventStream.subscribe(self, classOf[ChannelPersisted])
+
+  def receive = {
+    case persisted: ChannelPersisted =>
+      val start = System.currentTimeMillis()
+      val tmpFile = new File(backupFile.getAbsolutePath.concat(".tmp"))
+      databases.backup(tmpFile)
+      val result = tmpFile.renameTo(backupFile)
+      require(result, s"cannot rename $tmpFile to $backupFile")
+      val end = System.currentTimeMillis()
+      log.info(s"database backup triggered by channelId=${persisted.channelId} took ${end - start}ms")
+      backupScript_opt.foreach(backupScript => {
+        Try {
+          // run the script in the current thread and wait until it terminates
+          Process(backupScript).!
+        } match {
+          case Success(exitCode) => log.info(s"backup notify script $backupScript returned $exitCode")
+          case Failure(cause) => log.warning(s"cannot start backup notify script $backupScript:  $cause")
+        }
+      })
+  }
+}
+
+object BackupHandler {
+  // using this method is the only way to create a BackupHandler actor
+  // we make sure that it uses a custom bounded mailbox, and a custom pinned dispatcher (i.e our actor will have its own thread pool with 1 single thread)
+  def props(databases: Databases, backupFile: File, backupScript_opt: Option[String]) = Props(new BackupHandler(databases, backupFile, backupScript_opt)).withMailbox("eclair.backup-mailbox").withDispatcher("eclair.backup-dispatcher")
+}

eclair-core/src/main/scala/fr/acinq/eclair/db/Databases.scala

@@ -19,6 +19,7 @@ trait Databases {
 
   val pendingRelay: PendingRelayDb
 
+  def backup(file: File) : Unit
 }
 
 object Databases {
@@ -45,6 +46,12 @@ object Databases {
     override val peers = new SqlitePeersDb(eclairJdbc)
     override val payments = new SqlitePaymentsDb(eclairJdbc)
     override val pendingRelay = new SqlitePendingRelayDb(eclairJdbc)
+    override def backup(file: File): Unit = {
+      SqliteUtils.using(eclairJdbc.createStatement()) {
+        statement =>  {
+          statement.executeUpdate(s"backup to ${file.getAbsolutePath}")
+        }
+      }
+    }
   }
-
 }

eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqliteUtils.scala

@@ -109,7 +109,7 @@ object SqliteUtils {
     * Obtain an exclusive lock on a sqlite database. This is useful when we want to make sure that only one process
     * accesses the database file (see https://www.sqlite.org/pragma.html).
     *
-    * The lock will be kept until the database is closed, or if the locking mode is explicitely reset.
+    * The lock will be kept until the database is closed, or if the locking mode is explicitly reset.
     *
     * @param sqlite
     */

eclair-core/src/test/scala/fr/acinq/eclair/db/BackupHandlerSpec.scala

@@ -0,0 +1,38 @@
+package fr.acinq.eclair.db
+
+import java.io.File
+import java.sql.DriverManager
+import java.util.UUID
+
+import akka.actor.{ActorSystem, Props}
+import akka.testkit.TestKit
+import fr.acinq.eclair.channel.ChannelPersisted
+import fr.acinq.eclair.db.sqlite.SqliteChannelsDb
+import fr.acinq.eclair.{TestConstants, TestUtils, randomBytes32}
+import org.scalatest.FunSuiteLike
+
+import scala.concurrent.duration._
+
+class BackupHandlerSpec extends TestKit(ActorSystem("test")) with FunSuiteLike {
+
+  test("process backups") {
+    val db = TestConstants.inMemoryDb()
+    val wip = new File(TestUtils.BUILD_DIRECTORY, s"wip-${UUID.randomUUID()}")
+    val dest = new File(TestUtils.BUILD_DIRECTORY, s"backup-${UUID.randomUUID()}")
+    wip.deleteOnExit()
+    dest.deleteOnExit()
+    val channel = ChannelStateSpec.normal
+    db.channels.addOrUpdateChannel(channel)
+    assert(db.channels.listLocalChannels() == Seq(channel))
+
+    val handler = system.actorOf(BackupHandler.props(db, dest, None))
+    handler ! ChannelPersisted(null, TestConstants.Alice.nodeParams.nodeId, randomBytes32, null)
+    handler ! ChannelPersisted(null, TestConstants.Alice.nodeParams.nodeId, randomBytes32, null)
+    handler ! ChannelPersisted(null, TestConstants.Alice.nodeParams.nodeId, randomBytes32, null)
+    awaitCond(dest.exists(), 5 seconds)
+
+    val db1 = new SqliteChannelsDb(DriverManager.getConnection(s"jdbc:sqlite:$dest"))
+    val check = db1.listLocalChannels()
+    assert(check == Seq(channel))
+  }
+}