Txo state flyway (#1052)

* Add flyway migrations

* Make different project's migrations independent of each other

* Rework all AppConfig.initialize() to use migrations rather than what we were doing before

* TXO State migration

* Move to new file, drop old column

* Add block hash column

Co-authored-by: Chris Stewart <stewart.chris1234@gmail.com>

build.sbt

@@ -46,6 +46,7 @@ lazy val `bitcoin-s` = project
     core,
     coreTest,
     dbCommons,
+    dbCommonsTest,
     bitcoindRpc,
     bitcoindRpcTest,
     bench,
@@ -273,6 +274,13 @@ lazy val dbCommons = project
   )
   .dependsOn(core)
 
+lazy val dbCommonsTest = project
+  .in(file("db-commons-test"))
+  .settings(
+    name := "bitcoin-s-db-commons-test"
+  )
+  .dependsOn(testkit)
+
 lazy val zmq = project
   .in(file("zmq"))
   .settings(CommonSettings.prodSettings: _*)

chain/src/main/resources/chaindb/migration/V1__chain_db_baseline.sql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS "block_headers" ("height" INTEGER NOT NULL,"hash" VARCHAR(254) PRIMARY KEY NOT NULL,"version" INTEGER NOT NULL,"previous_block_hash" VARCHAR(254) NOT NULL,"merkle_root_hash" VARCHAR(254) NOT NULL,"time" INTEGER NOT NULL,"n_bits" INTEGER NOT NULL,"nonce" INTEGER NOT NULL,"hex" VARCHAR(254) NOT NULL);
+CREATE INDEX "block_headers_hash_index" on "block_headers" ("hash");
+CREATE INDEX "block_headers_height_index" on "block_headers" ("height");
+
+CREATE TABLE IF NOT EXISTS "cfheaders" ("hash" VARCHAR(254) PRIMARY KEY NOT NULL,"filter_hash" VARCHAR(254) NOT NULL,"previous_filter_header" VARCHAR(254) NOT NULL,"block_hash" VARCHAR(254) NOT NULL,"height" INTEGER NOT NULL);
+CREATE INDEX "cfheaders_block_hash_index" on "cfheaders" ("block_hash");
+CREATE INDEX "cfheaders_height_index" on "cfheaders" ("height");
+
+CREATE TABLE IF NOT EXISTS "cfilters" ("hash" VARCHAR(254) NOT NULL,"filter_type" INTEGER NOT NULL,"bytes" VARCHAR(254) NOT NULL,"height" INTEGER NOT NULL,"block_hash" VARCHAR(254) PRIMARY KEY NOT NULL);
+CREATE INDEX "cfilters_hash_index" on "cfilters" ("hash");
+CREATE INDEX "cfilters_height_index" on "cfilters" ("height");

chain/src/main/scala/org/bitcoins/chain/config/ChainAppConfig.scala

@@ -53,10 +53,11 @@ case class ChainAppConfig(
     * and inserts preliminary data like the genesis block header
     * */
   override def initialize()(implicit ec: ExecutionContext): Future[Unit] = {
-    val createdF = ChainDbManagement.createAll()(this, ec)
-    val isInitF = createdF.flatMap { _ =>
-      isInitialized()
-    }
+    val numMigrations = ChainDbManagement.migrate(this)
+
+    logger.info(s"Applied ${numMigrations} to chain project")
+
+    val isInitF = isInitialized()
     isInitF.flatMap { isInit =>
       if (isInit) {
         FutureUtil.unit
@@ -67,8 +68,7 @@ case class ChainAppConfig(
                                                 chain.genesisBlock.blockHeader)
         val blockHeaderDAO =
           BlockHeaderDAO()(ec = implicitly[ExecutionContext], appConfig = this)
-        val bhCreatedF =
-          createdF.flatMap(_ => blockHeaderDAO.create(genesisHeader))
+        val bhCreatedF = blockHeaderDAO.create(genesisHeader)
         bhCreatedF.flatMap { _ =>
           logger.info(s"Inserted genesis block header into DB")
           FutureUtil.unit

db-commons-test/src/test/scala/org/bitcoins/db/DbManagementTest.scala

@@ -0,0 +1,39 @@
+package org.bitcoins.db
+
+import com.typesafe.config.Config
+import org.bitcoins.chain.config.ChainAppConfig
+import org.bitcoins.chain.db.ChainDbManagement
+import org.bitcoins.node.config.NodeAppConfig
+import org.bitcoins.node.db.NodeDbManagement
+import org.bitcoins.testkit.BitcoinSTestAppConfig
+import org.bitcoins.testkit.BitcoinSTestAppConfig.ProjectType
+import org.bitcoins.testkit.util.BitcoinSUnitTest
+import org.bitcoins.wallet.config.WalletAppConfig
+import org.bitcoins.wallet.db.WalletDbManagement
+
+class DbManagementTest extends BitcoinSUnitTest {
+  def dbConfig(project: ProjectType): Config = {
+    BitcoinSTestAppConfig.configWithMemoryDb(Some(project))
+  }
+  it must "run migrations for chain db" in {
+    val chainAppConfig = ChainAppConfig(BitcoinSTestAppConfig.tmpDir(),
+      dbConfig(ProjectType.Chain))
+    val result = ChainDbManagement.migrate(chainAppConfig)
+    assert(result == 1)
+  }
+
+  it must "run migrations for wallet db" in {
+    val walletAppConfig = WalletAppConfig(BitcoinSTestAppConfig.tmpDir(),
+      dbConfig(ProjectType.Wallet))
+    val result = WalletDbManagement.migrate(walletAppConfig)
+    assert(result == 2)
+  }
+
+
+  it must "run migrations for node db" in {
+    val nodeAppConfig = NodeAppConfig(BitcoinSTestAppConfig.tmpDir(),
+      dbConfig(ProjectType.Node))
+    val result = NodeDbManagement.migrate(nodeAppConfig)
+    assert(result == 1)
+  }
+}

db-commons/src/main/scala/org/bitcoins/db/AppConfig.scala

@@ -125,6 +125,10 @@ abstract class AppConfig extends BitcoinSLogger {
     */
   protected[bitcoins] def moduleName: String
 
+  lazy val jdbcUrl: String = {
+    dbConfig.config.getString("db.url")
+  }
+
   /**
     * The configuration details for connecting/using the database for our projects
     * that require datbase connections

db-commons/src/main/scala/org/bitcoins/db/DbManagement.scala

@@ -1,5 +1,6 @@
 package org.bitcoins.db
 
+import org.flywaydb.core.Flyway
 import slick.jdbc.SQLiteProfile.api._
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -83,4 +84,19 @@ abstract class DbManagement extends DatabaseLogger {
     val result = database.run(table.schema.dropIfExists)
     result
   }
+
+  /** Executes migrations related to this database
+    *
+    * @see [[https://flywaydb.org/documentation/api/#programmatic-configuration-java]] */
+  def migrate(appConfig: AppConfig): Int = {
+    val url = appConfig.jdbcUrl
+    val username = ""
+    val password = ""
+    //appConfig.dbName is for the format 'walletdb.sqlite' or 'nodedb.sqlite' etc
+    //we need to remove the '.sqlite' suffix
+    val dbName = appConfig.dbName.split('.').head.mkString
+    val config = Flyway.configure().locations(s"classpath:${dbName}/migration/")
+    val flyway = config.dataSource(url, username, password).load
+    flyway.migrate()
+  }
 }

node/src/main/resources/nodedb/migration/V1__node_db_baseline.sql

@@ -0,0 +1 @@
+CREATE TABLE IF NOT EXISTS "broadcast_elements" ("txid" VARCHAR(254) NOT NULL UNIQUE,"tx_bytes" VARCHAR(254) NOT NULL,"id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL);

node/src/main/scala/org/bitcoins/node/config/NodeAppConfig.scala

@@ -1,14 +1,14 @@
 package org.bitcoins.node.config
 
-import com.typesafe.config.Config
-import org.bitcoins.db.AppConfig
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-import org.bitcoins.node.db.NodeDbManagement
-import scala.util.Failure
-import scala.util.Success
 import java.nio.file.Path
 
+import com.typesafe.config.Config
+import org.bitcoins.core.util.FutureUtil
+import org.bitcoins.db.AppConfig
+import org.bitcoins.node.db.NodeDbManagement
+
+import scala.concurrent.{ExecutionContext, Future}
+
 /** Configuration for the Bitcoin-S node
   * @param directory The data directory of the node
   * @param confs Optional sequence of configuration overrides
@@ -32,14 +32,11 @@ case class NodeAppConfig(
     */
   override def initialize()(implicit ec: ExecutionContext): Future[Unit] = {
     logger.debug(s"Initializing node setup")
-    val initF = NodeDbManagement.createAll()(config = this, ec)
-    initF.onComplete {
-      case Failure(err) =>
-        logger.error(s"Error when initializing node: ${err.getMessage}")
-      case Success(_) =>
-        logger.debug(s"Initializing node setup: done")
-    }
-    initF
+    val numMigrations = NodeDbManagement.migrate(this)
+
+    logger.info(s"Applied $numMigrations migrations fro the node project")
+
+    FutureUtil.unit
   }
 
   /**

node/src/main/scala/org/bitcoins/node/db/NodeDbManagement.scala

@@ -1,12 +1,13 @@
 package org.bitcoins.node.db
 
 import org.bitcoins.db.DbManagement
-import slick.lifted.TableQuery
 import org.bitcoins.node.models.BroadcastAbleTransactionTable
+import slick.lifted.TableQuery
 
 object NodeDbManagement extends DbManagement {
 
   private val txTable = TableQuery[BroadcastAbleTransactionTable]
 
   override val allTables = List(txTable)
+
 }

project/Deps.scala

@@ -24,6 +24,7 @@ object Deps {
     val asyncOldScalaV = "0.9.7"
     val asyncNewScalaV = "0.10.0"
 
+    val flywayV = "6.1.4"
     val postgresV = "9.4.1210"
     val akkaActorV = akkaStreamv
     val slickV = "3.3.2"
@@ -77,6 +78,7 @@ object Deps {
     val slickHikari = "com.typesafe.slick" %% "slick-hikaricp" % V.slickV
     val sqlite = "org.xerial" % "sqlite-jdbc" % V.sqliteV
     val postgres = "org.postgresql" % "postgresql" % V.postgresV
+    val flyway = "org.flywaydb" % "flyway-core" % V.flywayV
 
     // zero dep JSON library. Have to use different versiont to juggle
     // Scala 2.11/12/13
@@ -174,11 +176,14 @@ object Deps {
   )
 
   val dbCommons = List(
+    Compile.flyway,
     Compile.slick,
     Compile.sourcecode,
     Compile.logback,
     Compile.sqlite,
-    Compile.slickHikari
+    Compile.slickHikari,
+
+    Test.scalaTest
   )
 
   def cli(scalaVersion: String) = List(

wallet/src/main/resources/walletdb/migration/V1__wallet_db_baseline.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS "txo_spending_info" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,"tx_outpoint" VARCHAR(254) NOT NULL, "script_pub_key" VARCHAR(254) NOT NULL,"value" INTEGER NOT NULL,"hd_privkey_path" VARCHAR(254) NOT NULL,"redeem_script" VARCHAR(254),"script_witness" VARCHAR(254),"confirmations" INTEGER,"spent" INTEGER NOT NULL,"txid" VARCHAR(254) NOT NULL,"block_hash" VARCHAR(254),constraint "fk_scriptPubKey" foreign key("script_pub_key") references "addresses"("script_pub_key") on update NO ACTION on delete NO ACTION);
+
+CREATE TABLE IF NOT EXISTS "wallet_accounts" ("hd_purpose" INTEGER NOT NULL,"xpub" VARCHAR(254) NOT NULL,"coin" INTEGER NOT NULL,"account_index" INTEGER NOT NULL,constraint "pk_account" primary key("hd_purpose","coin","account_index"));
+
+CREATE TABLE IF NOT EXISTS "addresses" ("hd_purpose" INTEGER NOT NULL,"account_index" INTEGER NOT NULL,"hd_coin" INTEGER NOT NULL,"hd_chain_type" INTEGER NOT NULL,"address" VARCHAR(254) PRIMARY KEY NOT NULL,"script_witness" VARCHAR(254),"script_pub_key" VARCHAR(254) NOT NULL UNIQUE,"address_index" INTEGER NOT NULL,"pubkey" VARCHAR(254) NOT NULL,"hashed_pubkey" VARCHAR(254) NOT NULL,"script_type" VARCHAR(254) NOT NULL,constraint "fk_account" foreign key("hd_purpose","hd_coin","account_index") references "wallet_accounts"("hd_purpose","coin","account_index") on update NO ACTION on delete NO ACTION);

wallet/src/main/resources/walletdb/migration/V2__wallet_db_spent_to_txo_state.sql

@@ -0,0 +1,13 @@
+ALTER TABLE "txo_spending_info" ADD COLUMN "txo_state" VARCHAR(254);
+
+UPDATE "txo_spending_info" SET "txo_state" = "PendingConfirmationsSpent" WHERE "spent" = 1;
+UPDATE "txo_spending_info" SET "txo_state" = "PendingConfirmationsReceived" WHERE "spent" = 0;
+UPDATE "txo_spending_info" SET "confirmations" = 0 WHERE "confirmations" = NULL;
+
+-- This block drops the "spent" column
+CREATE TEMPORARY TABLE "txo_spending_info_backup" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,"tx_outpoint" VARCHAR(254) NOT NULL, "script_pub_key" VARCHAR(254) NOT NULL,"value" INTEGER NOT NULL,"hd_privkey_path" VARCHAR(254) NOT NULL,"redeem_script" VARCHAR(254),"script_witness" VARCHAR(254),"confirmations" INTEGER,"txid" VARCHAR(254) NOT NULL,"block_hash" VARCHAR(254), "txo_state" VARCHAR(254) NOT NULL, constraint "fk_scriptPubKey" foreign key("script_pub_key") references "addresses"("script_pub_key"));
+INSERT INTO "txo_spending_info_backup" SELECT "id", "tx_outpoint", "script_pub_key", "value", "hd_privkey_path", "redeem_script", "script_witness", "confirmations", "txid","block_hash", "txo_state" FROM "txo_spending_info";
+DROP TABLE "txo_spending_info";
+CREATE TABLE "txo_spending_info" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,"tx_outpoint" VARCHAR(254) NOT NULL, "script_pub_key" VARCHAR(254) NOT NULL,"value" INTEGER NOT NULL,"hd_privkey_path" VARCHAR(254) NOT NULL,"redeem_script" VARCHAR(254),"script_witness" VARCHAR(254),"confirmations" INTEGER,"txid" VARCHAR(254) NOT NULL,"block_hash" VARCHAR(254), "txo_state" VARCHAR(254) NOT NULL, constraint "fk_scriptPubKey" foreign key("script_pub_key") references "addresses"("script_pub_key") on update NO ACTION on delete NO ACTION);
+INSERT INTO "txo_spending_info" SELECT "id", "tx_outpoint", "script_pub_key", "value", "hd_privkey_path", "redeem_script", "script_witness", "confirmations", "txid","block_hash", "txo_state" FROM "txo_spending_info_backup";
+DROP TABLE "txo_spending_info_backup";

wallet/src/main/scala/org/bitcoins/wallet/config/WalletAppConfig.scala

@@ -3,20 +3,13 @@ package org.bitcoins.wallet.config
 import java.nio.file.{Files, Path}
 
 import com.typesafe.config.Config
-import org.bitcoins.core.hd.{
-  AddressType,
-  HDAccount,
-  HDCoin,
-  HDCoinType,
-  HDPurpose,
-  HDPurposes
-}
+import org.bitcoins.core.hd._
+import org.bitcoins.core.util.FutureUtil
 import org.bitcoins.db.AppConfig
 import org.bitcoins.keymanager.{KeyManagerParams, WalletStorage}
 import org.bitcoins.wallet.db.WalletDbManagement
 
 import scala.concurrent.{ExecutionContext, Future}
-import scala.util.{Failure, Success}
 
 /** Configuration for the Bitcoin-S wallet
   * @param directory The data directory of the wallet
@@ -75,17 +68,13 @@ case class WalletAppConfig(
       Files.createDirectories(datadir)
     }
 
-    val initF = {
-      WalletDbManagement.createAll()(this, ec)
-    }
-    initF.onComplete {
-      case Failure(exception) =>
-        logger.error(s"Error on wallet setup: ${exception.getMessage}")
-      case Success(_) =>
-        logger.debug(s"Initializing wallet setup: done")
+    val numMigrations = {
+      WalletDbManagement.migrate(this)
     }
 
-    initF
+    logger.info(s"Applied $numMigrations to the wallet project")
+
+    FutureUtil.unit
   }
 
   /** The path to our encrypted mnemonic seed */