From ed428bd56cd98514b55620775fa9e245ad0d4c23 Mon Sep 17 00:00:00 2001 From: Ben Carman Date: Wed, 22 Jan 2020 15:34:36 -0600 Subject: [PATCH] Txo state flyway (#1052) * Add flyway migrations * Make different project's migrations independent of each other * Rework all AppConfig.initialize() to use migrations rather than what we were doing before * TXO State migration * Move to new file, drop old column * Add block hash column Co-authored-by: Chris Stewart --- build.sbt | 8 ++++ .../migration/V1__chain_db_baseline.sql | 11 ++++++ .../chain/config/ChainAppConfig.scala | 12 +++--- .../org/bitcoins/db/DbManagementTest.scala | 39 +++++++++++++++++++ .../scala/org/bitcoins/db/AppConfig.scala | 4 ++ .../scala/org/bitcoins/db/DbManagement.scala | 16 ++++++++ .../nodedb/migration/V1__node_db_baseline.sql | 1 + .../bitcoins/node/config/NodeAppConfig.scala | 27 ++++++------- .../bitcoins/node/db/NodeDbManagement.scala | 3 +- project/Deps.scala | 7 +++- .../migration/V1__wallet_db_baseline.sql | 5 +++ .../V2__wallet_db_spent_to_txo_state.sql | 13 +++++++ .../wallet/config/WalletAppConfig.scala | 25 ++++-------- 13 files changed, 130 insertions(+), 41 deletions(-) create mode 100644 chain/src/main/resources/chaindb/migration/V1__chain_db_baseline.sql create mode 100644 db-commons-test/src/test/scala/org/bitcoins/db/DbManagementTest.scala create mode 100644 node/src/main/resources/nodedb/migration/V1__node_db_baseline.sql create mode 100644 wallet/src/main/resources/walletdb/migration/V1__wallet_db_baseline.sql create mode 100644 wallet/src/main/resources/walletdb/migration/V2__wallet_db_spent_to_txo_state.sql diff --git a/build.sbt b/build.sbt index 4c20a17735..e21c45bd2f 100644 --- a/build.sbt +++ b/build.sbt @@ -46,6 +46,7 @@ lazy val `bitcoin-s` = project core, coreTest, dbCommons, + dbCommonsTest, bitcoindRpc, bitcoindRpcTest, bench, @@ -273,6 +274,13 @@ lazy val dbCommons = project ) .dependsOn(core) +lazy val dbCommonsTest = project + .in(file("db-commons-test")) + .settings( + name := "bitcoin-s-db-commons-test" + ) + .dependsOn(testkit) + lazy val zmq = project .in(file("zmq")) .settings(CommonSettings.prodSettings: _*) diff --git a/chain/src/main/resources/chaindb/migration/V1__chain_db_baseline.sql b/chain/src/main/resources/chaindb/migration/V1__chain_db_baseline.sql new file mode 100644 index 0000000000..5bf06fcec0 --- /dev/null +++ b/chain/src/main/resources/chaindb/migration/V1__chain_db_baseline.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS "block_headers" ("height" INTEGER NOT NULL,"hash" VARCHAR(254) PRIMARY KEY NOT NULL,"version" INTEGER NOT NULL,"previous_block_hash" VARCHAR(254) NOT NULL,"merkle_root_hash" VARCHAR(254) NOT NULL,"time" INTEGER NOT NULL,"n_bits" INTEGER NOT NULL,"nonce" INTEGER NOT NULL,"hex" VARCHAR(254) NOT NULL); +CREATE INDEX "block_headers_hash_index" on "block_headers" ("hash"); +CREATE INDEX "block_headers_height_index" on "block_headers" ("height"); + +CREATE TABLE IF NOT EXISTS "cfheaders" ("hash" VARCHAR(254) PRIMARY KEY NOT NULL,"filter_hash" VARCHAR(254) NOT NULL,"previous_filter_header" VARCHAR(254) NOT NULL,"block_hash" VARCHAR(254) NOT NULL,"height" INTEGER NOT NULL); +CREATE INDEX "cfheaders_block_hash_index" on "cfheaders" ("block_hash"); +CREATE INDEX "cfheaders_height_index" on "cfheaders" ("height"); + +CREATE TABLE IF NOT EXISTS "cfilters" ("hash" VARCHAR(254) NOT NULL,"filter_type" INTEGER NOT NULL,"bytes" VARCHAR(254) NOT NULL,"height" INTEGER NOT NULL,"block_hash" VARCHAR(254) PRIMARY KEY NOT NULL); +CREATE INDEX "cfilters_hash_index" on "cfilters" ("hash"); +CREATE INDEX "cfilters_height_index" on "cfilters" ("height"); diff --git a/chain/src/main/scala/org/bitcoins/chain/config/ChainAppConfig.scala b/chain/src/main/scala/org/bitcoins/chain/config/ChainAppConfig.scala index f0bd8fdf39..4c75b4f891 100644 --- a/chain/src/main/scala/org/bitcoins/chain/config/ChainAppConfig.scala +++ b/chain/src/main/scala/org/bitcoins/chain/config/ChainAppConfig.scala @@ -53,10 +53,11 @@ case class ChainAppConfig( * and inserts preliminary data like the genesis block header * */ override def initialize()(implicit ec: ExecutionContext): Future[Unit] = { - val createdF = ChainDbManagement.createAll()(this, ec) - val isInitF = createdF.flatMap { _ => - isInitialized() - } + val numMigrations = ChainDbManagement.migrate(this) + + logger.info(s"Applied ${numMigrations} to chain project") + + val isInitF = isInitialized() isInitF.flatMap { isInit => if (isInit) { FutureUtil.unit @@ -67,8 +68,7 @@ case class ChainAppConfig( chain.genesisBlock.blockHeader) val blockHeaderDAO = BlockHeaderDAO()(ec = implicitly[ExecutionContext], appConfig = this) - val bhCreatedF = - createdF.flatMap(_ => blockHeaderDAO.create(genesisHeader)) + val bhCreatedF = blockHeaderDAO.create(genesisHeader) bhCreatedF.flatMap { _ => logger.info(s"Inserted genesis block header into DB") FutureUtil.unit diff --git a/db-commons-test/src/test/scala/org/bitcoins/db/DbManagementTest.scala b/db-commons-test/src/test/scala/org/bitcoins/db/DbManagementTest.scala new file mode 100644 index 0000000000..2e13672a8f --- /dev/null +++ b/db-commons-test/src/test/scala/org/bitcoins/db/DbManagementTest.scala @@ -0,0 +1,39 @@ +package org.bitcoins.db + +import com.typesafe.config.Config +import org.bitcoins.chain.config.ChainAppConfig +import org.bitcoins.chain.db.ChainDbManagement +import org.bitcoins.node.config.NodeAppConfig +import org.bitcoins.node.db.NodeDbManagement +import org.bitcoins.testkit.BitcoinSTestAppConfig +import org.bitcoins.testkit.BitcoinSTestAppConfig.ProjectType +import org.bitcoins.testkit.util.BitcoinSUnitTest +import org.bitcoins.wallet.config.WalletAppConfig +import org.bitcoins.wallet.db.WalletDbManagement + +class DbManagementTest extends BitcoinSUnitTest { + def dbConfig(project: ProjectType): Config = { + BitcoinSTestAppConfig.configWithMemoryDb(Some(project)) + } + it must "run migrations for chain db" in { + val chainAppConfig = ChainAppConfig(BitcoinSTestAppConfig.tmpDir(), + dbConfig(ProjectType.Chain)) + val result = ChainDbManagement.migrate(chainAppConfig) + assert(result == 1) + } + + it must "run migrations for wallet db" in { + val walletAppConfig = WalletAppConfig(BitcoinSTestAppConfig.tmpDir(), + dbConfig(ProjectType.Wallet)) + val result = WalletDbManagement.migrate(walletAppConfig) + assert(result == 2) + } + + + it must "run migrations for node db" in { + val nodeAppConfig = NodeAppConfig(BitcoinSTestAppConfig.tmpDir(), + dbConfig(ProjectType.Node)) + val result = NodeDbManagement.migrate(nodeAppConfig) + assert(result == 1) + } +} diff --git a/db-commons/src/main/scala/org/bitcoins/db/AppConfig.scala b/db-commons/src/main/scala/org/bitcoins/db/AppConfig.scala index 919c4b57f6..870bf519dd 100644 --- a/db-commons/src/main/scala/org/bitcoins/db/AppConfig.scala +++ b/db-commons/src/main/scala/org/bitcoins/db/AppConfig.scala @@ -125,6 +125,10 @@ abstract class AppConfig extends BitcoinSLogger { */ protected[bitcoins] def moduleName: String + lazy val jdbcUrl: String = { + dbConfig.config.getString("db.url") + } + /** * The configuration details for connecting/using the database for our projects * that require datbase connections diff --git a/db-commons/src/main/scala/org/bitcoins/db/DbManagement.scala b/db-commons/src/main/scala/org/bitcoins/db/DbManagement.scala index f034a63c48..5b5a4343c3 100644 --- a/db-commons/src/main/scala/org/bitcoins/db/DbManagement.scala +++ b/db-commons/src/main/scala/org/bitcoins/db/DbManagement.scala @@ -1,5 +1,6 @@ package org.bitcoins.db +import org.flywaydb.core.Flyway import slick.jdbc.SQLiteProfile.api._ import scala.concurrent.{ExecutionContext, Future} @@ -83,4 +84,19 @@ abstract class DbManagement extends DatabaseLogger { val result = database.run(table.schema.dropIfExists) result } + + /** Executes migrations related to this database + * + * @see [[https://flywaydb.org/documentation/api/#programmatic-configuration-java]] */ + def migrate(appConfig: AppConfig): Int = { + val url = appConfig.jdbcUrl + val username = "" + val password = "" + //appConfig.dbName is for the format 'walletdb.sqlite' or 'nodedb.sqlite' etc + //we need to remove the '.sqlite' suffix + val dbName = appConfig.dbName.split('.').head.mkString + val config = Flyway.configure().locations(s"classpath:${dbName}/migration/") + val flyway = config.dataSource(url, username, password).load + flyway.migrate() + } } diff --git a/node/src/main/resources/nodedb/migration/V1__node_db_baseline.sql b/node/src/main/resources/nodedb/migration/V1__node_db_baseline.sql new file mode 100644 index 0000000000..58cb5e73c1 --- /dev/null +++ b/node/src/main/resources/nodedb/migration/V1__node_db_baseline.sql @@ -0,0 +1 @@ +CREATE TABLE IF NOT EXISTS "broadcast_elements" ("txid" VARCHAR(254) NOT NULL UNIQUE,"tx_bytes" VARCHAR(254) NOT NULL,"id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL); diff --git a/node/src/main/scala/org/bitcoins/node/config/NodeAppConfig.scala b/node/src/main/scala/org/bitcoins/node/config/NodeAppConfig.scala index 53bf2a343b..43e2276701 100644 --- a/node/src/main/scala/org/bitcoins/node/config/NodeAppConfig.scala +++ b/node/src/main/scala/org/bitcoins/node/config/NodeAppConfig.scala @@ -1,14 +1,14 @@ package org.bitcoins.node.config -import com.typesafe.config.Config -import org.bitcoins.db.AppConfig -import scala.concurrent.ExecutionContext -import scala.concurrent.Future -import org.bitcoins.node.db.NodeDbManagement -import scala.util.Failure -import scala.util.Success import java.nio.file.Path +import com.typesafe.config.Config +import org.bitcoins.core.util.FutureUtil +import org.bitcoins.db.AppConfig +import org.bitcoins.node.db.NodeDbManagement + +import scala.concurrent.{ExecutionContext, Future} + /** Configuration for the Bitcoin-S node * @param directory The data directory of the node * @param confs Optional sequence of configuration overrides @@ -32,14 +32,11 @@ case class NodeAppConfig( */ override def initialize()(implicit ec: ExecutionContext): Future[Unit] = { logger.debug(s"Initializing node setup") - val initF = NodeDbManagement.createAll()(config = this, ec) - initF.onComplete { - case Failure(err) => - logger.error(s"Error when initializing node: ${err.getMessage}") - case Success(_) => - logger.debug(s"Initializing node setup: done") - } - initF + val numMigrations = NodeDbManagement.migrate(this) + + logger.info(s"Applied $numMigrations migrations fro the node project") + + FutureUtil.unit } /** diff --git a/node/src/main/scala/org/bitcoins/node/db/NodeDbManagement.scala b/node/src/main/scala/org/bitcoins/node/db/NodeDbManagement.scala index 2f26102892..8f45e8abe7 100644 --- a/node/src/main/scala/org/bitcoins/node/db/NodeDbManagement.scala +++ b/node/src/main/scala/org/bitcoins/node/db/NodeDbManagement.scala @@ -1,12 +1,13 @@ package org.bitcoins.node.db import org.bitcoins.db.DbManagement -import slick.lifted.TableQuery import org.bitcoins.node.models.BroadcastAbleTransactionTable +import slick.lifted.TableQuery object NodeDbManagement extends DbManagement { private val txTable = TableQuery[BroadcastAbleTransactionTable] override val allTables = List(txTable) + } diff --git a/project/Deps.scala b/project/Deps.scala index 003db922f8..1e674902eb 100644 --- a/project/Deps.scala +++ b/project/Deps.scala @@ -24,6 +24,7 @@ object Deps { val asyncOldScalaV = "0.9.7" val asyncNewScalaV = "0.10.0" + val flywayV = "6.1.4" val postgresV = "9.4.1210" val akkaActorV = akkaStreamv val slickV = "3.3.2" @@ -77,6 +78,7 @@ object Deps { val slickHikari = "com.typesafe.slick" %% "slick-hikaricp" % V.slickV val sqlite = "org.xerial" % "sqlite-jdbc" % V.sqliteV val postgres = "org.postgresql" % "postgresql" % V.postgresV + val flyway = "org.flywaydb" % "flyway-core" % V.flywayV // zero dep JSON library. Have to use different versiont to juggle // Scala 2.11/12/13 @@ -174,11 +176,14 @@ object Deps { ) val dbCommons = List( + Compile.flyway, Compile.slick, Compile.sourcecode, Compile.logback, Compile.sqlite, - Compile.slickHikari + Compile.slickHikari, + + Test.scalaTest ) def cli(scalaVersion: String) = List( diff --git a/wallet/src/main/resources/walletdb/migration/V1__wallet_db_baseline.sql b/wallet/src/main/resources/walletdb/migration/V1__wallet_db_baseline.sql new file mode 100644 index 0000000000..5047e9f0c3 --- /dev/null +++ b/wallet/src/main/resources/walletdb/migration/V1__wallet_db_baseline.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS "txo_spending_info" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,"tx_outpoint" VARCHAR(254) NOT NULL, "script_pub_key" VARCHAR(254) NOT NULL,"value" INTEGER NOT NULL,"hd_privkey_path" VARCHAR(254) NOT NULL,"redeem_script" VARCHAR(254),"script_witness" VARCHAR(254),"confirmations" INTEGER,"spent" INTEGER NOT NULL,"txid" VARCHAR(254) NOT NULL,"block_hash" VARCHAR(254),constraint "fk_scriptPubKey" foreign key("script_pub_key") references "addresses"("script_pub_key") on update NO ACTION on delete NO ACTION); + +CREATE TABLE IF NOT EXISTS "wallet_accounts" ("hd_purpose" INTEGER NOT NULL,"xpub" VARCHAR(254) NOT NULL,"coin" INTEGER NOT NULL,"account_index" INTEGER NOT NULL,constraint "pk_account" primary key("hd_purpose","coin","account_index")); + +CREATE TABLE IF NOT EXISTS "addresses" ("hd_purpose" INTEGER NOT NULL,"account_index" INTEGER NOT NULL,"hd_coin" INTEGER NOT NULL,"hd_chain_type" INTEGER NOT NULL,"address" VARCHAR(254) PRIMARY KEY NOT NULL,"script_witness" VARCHAR(254),"script_pub_key" VARCHAR(254) NOT NULL UNIQUE,"address_index" INTEGER NOT NULL,"pubkey" VARCHAR(254) NOT NULL,"hashed_pubkey" VARCHAR(254) NOT NULL,"script_type" VARCHAR(254) NOT NULL,constraint "fk_account" foreign key("hd_purpose","hd_coin","account_index") references "wallet_accounts"("hd_purpose","coin","account_index") on update NO ACTION on delete NO ACTION); \ No newline at end of file diff --git a/wallet/src/main/resources/walletdb/migration/V2__wallet_db_spent_to_txo_state.sql b/wallet/src/main/resources/walletdb/migration/V2__wallet_db_spent_to_txo_state.sql new file mode 100644 index 0000000000..f8cba70ff8 --- /dev/null +++ b/wallet/src/main/resources/walletdb/migration/V2__wallet_db_spent_to_txo_state.sql @@ -0,0 +1,13 @@ +ALTER TABLE "txo_spending_info" ADD COLUMN "txo_state" VARCHAR(254); + +UPDATE "txo_spending_info" SET "txo_state" = "PendingConfirmationsSpent" WHERE "spent" = 1; +UPDATE "txo_spending_info" SET "txo_state" = "PendingConfirmationsReceived" WHERE "spent" = 0; +UPDATE "txo_spending_info" SET "confirmations" = 0 WHERE "confirmations" = NULL; + +-- This block drops the "spent" column +CREATE TEMPORARY TABLE "txo_spending_info_backup" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,"tx_outpoint" VARCHAR(254) NOT NULL, "script_pub_key" VARCHAR(254) NOT NULL,"value" INTEGER NOT NULL,"hd_privkey_path" VARCHAR(254) NOT NULL,"redeem_script" VARCHAR(254),"script_witness" VARCHAR(254),"confirmations" INTEGER,"txid" VARCHAR(254) NOT NULL,"block_hash" VARCHAR(254), "txo_state" VARCHAR(254) NOT NULL, constraint "fk_scriptPubKey" foreign key("script_pub_key") references "addresses"("script_pub_key")); +INSERT INTO "txo_spending_info_backup" SELECT "id", "tx_outpoint", "script_pub_key", "value", "hd_privkey_path", "redeem_script", "script_witness", "confirmations", "txid","block_hash", "txo_state" FROM "txo_spending_info"; +DROP TABLE "txo_spending_info"; +CREATE TABLE "txo_spending_info" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,"tx_outpoint" VARCHAR(254) NOT NULL, "script_pub_key" VARCHAR(254) NOT NULL,"value" INTEGER NOT NULL,"hd_privkey_path" VARCHAR(254) NOT NULL,"redeem_script" VARCHAR(254),"script_witness" VARCHAR(254),"confirmations" INTEGER,"txid" VARCHAR(254) NOT NULL,"block_hash" VARCHAR(254), "txo_state" VARCHAR(254) NOT NULL, constraint "fk_scriptPubKey" foreign key("script_pub_key") references "addresses"("script_pub_key") on update NO ACTION on delete NO ACTION); +INSERT INTO "txo_spending_info" SELECT "id", "tx_outpoint", "script_pub_key", "value", "hd_privkey_path", "redeem_script", "script_witness", "confirmations", "txid","block_hash", "txo_state" FROM "txo_spending_info_backup"; +DROP TABLE "txo_spending_info_backup"; diff --git a/wallet/src/main/scala/org/bitcoins/wallet/config/WalletAppConfig.scala b/wallet/src/main/scala/org/bitcoins/wallet/config/WalletAppConfig.scala index 50175d58d5..5b0330f0b9 100644 --- a/wallet/src/main/scala/org/bitcoins/wallet/config/WalletAppConfig.scala +++ b/wallet/src/main/scala/org/bitcoins/wallet/config/WalletAppConfig.scala @@ -3,20 +3,13 @@ package org.bitcoins.wallet.config import java.nio.file.{Files, Path} import com.typesafe.config.Config -import org.bitcoins.core.hd.{ - AddressType, - HDAccount, - HDCoin, - HDCoinType, - HDPurpose, - HDPurposes -} +import org.bitcoins.core.hd._ +import org.bitcoins.core.util.FutureUtil import org.bitcoins.db.AppConfig import org.bitcoins.keymanager.{KeyManagerParams, WalletStorage} import org.bitcoins.wallet.db.WalletDbManagement import scala.concurrent.{ExecutionContext, Future} -import scala.util.{Failure, Success} /** Configuration for the Bitcoin-S wallet * @param directory The data directory of the wallet @@ -75,17 +68,13 @@ case class WalletAppConfig( Files.createDirectories(datadir) } - val initF = { - WalletDbManagement.createAll()(this, ec) - } - initF.onComplete { - case Failure(exception) => - logger.error(s"Error on wallet setup: ${exception.getMessage}") - case Success(_) => - logger.debug(s"Initializing wallet setup: done") + val numMigrations = { + WalletDbManagement.migrate(this) } - initF + logger.info(s"Applied $numMigrations to the wallet project") + + FutureUtil.unit } /** The path to our encrypted mnemonic seed */