PostgreSQL support (#1315)

.gitignore

@@ -82,3 +82,6 @@ website/static/api
 #bloop 
 .bloop/
 .metals/
+
+#OSX
+.DS_Store

.travis.yml

@@ -28,6 +28,13 @@ matrix:
         - TEST_COMMAND="coreTest/test cryptoTest/test"
       scala:
         - 2.13.2
+    - os: linux
+      name: "PostgreSQL tests"
+      env:
+        - PG_ENABLED="1"
+        - TEST_COMMAND="dbCommonsTest/test chainTest/test nodeTest/test walletTest/test"
+      scala:
+        - 2.13.2
     - os: osx
       name: "macOS bitcoind and eclair tests"
       env:

chain-test/src/test/scala/org/bitcoins/chain/blockchain/BitcoindChainHandlerViaZmqTest.scala

@@ -2,13 +2,11 @@ package org.bitcoins.chain.blockchain
 
 import akka.actor.ActorSystem
 import org.bitcoins.rpc.util.RpcUtil
-import org.bitcoins.testkit.chain.ChainUnitTest
+import org.bitcoins.testkit.chain.{ChainDbUnitTest, ChainUnitTest}
 import org.bitcoins.testkit.chain.fixture.BitcoindChainHandlerViaZmq
 import org.scalatest.FutureOutcome
 
-import scala.concurrent.Future
-
-class BitcoindChainHandlerViaZmqTest extends ChainUnitTest {
+class BitcoindChainHandlerViaZmqTest extends ChainDbUnitTest {
 
   override type FixtureParam = BitcoindChainHandlerViaZmq
 

chain-test/src/test/scala/org/bitcoins/chain/blockchain/BlockchainTest.scala

@@ -1,7 +1,7 @@
 package org.bitcoins.chain.blockchain
 
 import akka.actor.ActorSystem
-import org.bitcoins.chain.models.{BlockHeaderDAO, BlockHeaderDb}
+import org.bitcoins.chain.models.BlockHeaderDb
 import org.bitcoins.testkit.chain.fixture.ChainFixture
 import org.bitcoins.testkit.chain.{BlockHeaderHelper, ChainUnitTest}
 import org.scalatest.FutureOutcome

chain-test/src/test/scala/org/bitcoins/chain/blockchain/ChainHandlerTest.scala

@@ -15,10 +15,10 @@ import org.bitcoins.crypto.{
   DoubleSha256DigestBE,
   ECPrivateKey
 }
-import org.bitcoins.testkit.BitcoinSTestAppConfig
 import org.bitcoins.testkit.chain.fixture.ChainFixtureTag
 import org.bitcoins.testkit.chain.{
   BlockHeaderHelper,
+  ChainDbUnitTest,
   ChainTestUtil,
   ChainUnitTest
 }
@@ -29,20 +29,14 @@ import play.api.libs.json.Json
 import scala.concurrent.Future
 import scala.io.BufferedSource
 
-class ChainHandlerTest extends ChainUnitTest {
+class ChainHandlerTest extends ChainDbUnitTest {
 
   override type FixtureParam = ChainHandler
 
   implicit override val system = ActorSystem("ChainUnitTest")
 
   // we're working with mainnet data
-  implicit override lazy val appConfig: ChainAppConfig = {
-    import BitcoinSTestAppConfig.ProjectType
-
-    val memoryDb =
-      BitcoinSTestAppConfig.configWithMemoryDb(Some(ProjectType.Chain))
-    mainnetAppConfig.withOverrides(memoryDb)
-  }
+  implicit override lazy val appConfig: ChainAppConfig = mainnetAppConfig
 
   val source: BufferedSource = FileUtil.getFileAsSource("block_headers.json")
   val arrStr: String = source.getLines.next

chain-test/src/test/scala/org/bitcoins/chain/blockchain/sync/ChainSyncTest.scala

@@ -4,13 +4,13 @@ import akka.actor.ActorSystem
 import org.bitcoins.chain.api.ChainApi
 import org.bitcoins.chain.blockchain.ChainHandler
 import org.bitcoins.crypto.DoubleSha256DigestBE
-import org.bitcoins.testkit.chain.{ChainUnitTest, SyncUtil}
 import org.bitcoins.testkit.chain.fixture.BitcoindChainHandlerViaRpc
+import org.bitcoins.testkit.chain.{ChainDbUnitTest, SyncUtil}
 import org.scalatest.FutureOutcome
 
 import scala.concurrent.Future
 
-class ChainSyncTest extends ChainUnitTest {
+class ChainSyncTest extends ChainDbUnitTest {
   override type FixtureParam = BitcoindChainHandlerViaRpc
 
   implicit override val system = ActorSystem(

chain-test/src/test/scala/org/bitcoins/chain/blockchain/sync/FilterSyncTest.scala

@@ -2,15 +2,15 @@ package org.bitcoins.chain.blockchain.sync
 
 import org.bitcoins.chain.api.ChainApi
 import org.bitcoins.chain.blockchain.ChainHandler
-import org.bitcoins.core.gcs.{FilterType, GolombFilter}
+import org.bitcoins.core.gcs.FilterType
 import org.bitcoins.core.protocol.blockchain.BlockHeader
 import org.bitcoins.testkit.chain.fixture.BitcoindV19ChainHandler
-import org.bitcoins.testkit.chain.{ChainUnitTest, SyncUtil}
+import org.bitcoins.testkit.chain.{ChainDbUnitTest, SyncUtil}
 import org.scalatest.FutureOutcome
 
 import scala.concurrent.Future
 
-class FilterSyncTest extends ChainUnitTest {
+class FilterSyncTest extends ChainDbUnitTest {
 
   override type FixtureParam = BitcoindV19ChainHandler
 
@@ -53,7 +53,7 @@ class FilterSyncTest extends ChainUnitTest {
 
   it must "sync a bunch of filter headers from an external data source" in {
     fixture =>
-      val BitcoindV19ChainHandler(bitcoind, chainHandler) = fixture
+      val BitcoindV19ChainHandler(bitcoind, _) = fixture
 
       val numBlocks = 100
       val generatedBlocksF = for {
@@ -75,7 +75,7 @@ class FilterSyncTest extends ChainUnitTest {
 
   it must "be able to call filterSync() and not fail when nothing has happened" in {
     fixture =>
-      val BitcoindV19ChainHandler(bitcoind, chainHandler) = fixture
+      val BitcoindV19ChainHandler(bitcoind, _) = fixture
 
       val generated1BlockF = for {
         addr <- bitcoind.getNewAddress

chain-test/src/test/scala/org/bitcoins/chain/models/BlockHeaderDAOTest.scala

@@ -4,7 +4,11 @@ import akka.actor.ActorSystem
 import org.bitcoins.core.number.UInt32
 import org.bitcoins.core.util.TimeUtil
 import org.bitcoins.crypto.DoubleSha256DigestBE
-import org.bitcoins.testkit.chain.{BlockHeaderHelper, ChainUnitTest}
+import org.bitcoins.testkit.chain.{
+  BlockHeaderHelper,
+  ChainDbUnitTest,
+  ChainUnitTest
+}
 import org.scalatest.FutureOutcome
 
 import scala.concurrent.Future
@@ -12,7 +16,7 @@ import scala.concurrent.Future
 /**
   * Created by chris on 9/8/16.
   */
-class BlockHeaderDAOTest extends ChainUnitTest {
+class BlockHeaderDAOTest extends ChainDbUnitTest {
 
   override type FixtureParam = BlockHeaderDAO
 
@@ -222,7 +226,8 @@ class BlockHeaderDAOTest extends ChainUnitTest {
 
       getHeightF.map {
         case headers =>
-          assert(headers == Seq(blockHeader, blockHeader1))
+          assert(headers.toSet.size == 2)
+          assert(headers.toSet == Set(blockHeader, blockHeader1))
       }
   }
 

chain-test/src/test/scala/org/bitcoins/chain/pow/BitcoinPowTest.scala

@@ -3,26 +3,23 @@ package org.bitcoins.chain.pow
 import akka.actor.ActorSystem
 import org.bitcoins.chain.blockchain.Blockchain
 import org.bitcoins.chain.config.ChainAppConfig
-import org.bitcoins.chain.models.BlockHeaderDAO
 import org.bitcoins.core.protocol.blockchain.MainNetChainParams
 import org.bitcoins.testkit.chain.fixture.{ChainFixture, ChainFixtureTag}
-import org.bitcoins.testkit.chain.{ChainTestUtil, ChainUnitTest}
+import org.bitcoins.testkit.chain.{
+  ChainDbUnitTest,
+  ChainTestUtil,
+  ChainUnitTest
+}
 import org.scalatest.FutureOutcome
 
 import scala.concurrent.Future
-import org.bitcoins.server.BitcoinSAppConfig
-import org.bitcoins.testkit.BitcoinSTestAppConfig
 
-class BitcoinPowTest extends ChainUnitTest {
+class BitcoinPowTest extends ChainDbUnitTest {
 
   override type FixtureParam = ChainFixture
 
-  implicit override lazy val appConfig: ChainAppConfig = {
-    import BitcoinSTestAppConfig.ProjectType
-    val memoryDb =
-      BitcoinSTestAppConfig.configWithMemoryDb(Some(ProjectType.Chain))
-    mainnetAppConfig.withOverrides(memoryDb)
-  }
+  // we're working with mainnet data
+  implicit override lazy val appConfig: ChainAppConfig = mainnetAppConfig
 
   override def withFixture(test: OneArgAsyncTest): FutureOutcome =
     withChainFixture(test)

chain-test/src/test/scala/org/bitcoins/chain/validation/TipValidationTest.scala

@@ -1,27 +1,14 @@
 package org.bitcoins.chain.validation
 
 import akka.actor.ActorSystem
-import org.bitcoins.chain.db.ChainDbManagement
-import org.bitcoins.chain.models.{
-  BlockHeaderDAO,
-  BlockHeaderDb,
-  BlockHeaderDbHelper
-}
+import org.bitcoins.chain.blockchain.Blockchain
+import org.bitcoins.chain.config.ChainAppConfig
+import org.bitcoins.chain.models.{BlockHeaderDAO, BlockHeaderDbHelper}
 import org.bitcoins.core.protocol.blockchain.BlockHeader
-import org.bitcoins.testkit.chain.{
-  BlockHeaderHelper,
-  ChainTestUtil,
-  ChainUnitTest
-}
+import org.bitcoins.testkit.chain.{BlockHeaderHelper, ChainDbUnitTest}
 import org.scalatest.{Assertion, FutureOutcome}
 
-import scala.concurrent.Future
-import org.bitcoins.chain.config.ChainAppConfig
-import com.typesafe.config.ConfigFactory
-import org.bitcoins.chain.blockchain.Blockchain
-import org.bitcoins.server.BitcoinSAppConfig
-
-class TipValidationTest extends ChainUnitTest {
+class TipValidationTest extends ChainDbUnitTest {
 
   override type FixtureParam = BlockHeaderDAO
 

chain/src/main/resources/postgresql/chain/migration/V1__chain_db_baseline.sql

@@ -0,0 +1,13 @@
+CREATE TABLE IF NOT EXISTS block_headers (height BIGINT NOT NULL,hash VARCHAR(254) PRIMARY KEY NOT NULL,version INTEGER NOT NULL,previous_block_hash VARCHAR(254) NOT NULL,merkle_root_hash VARCHAR(254) NOT NULL,time BIGINT NOT NULL,n_bits BIGINT NOT NULL,nonce BIGINT NOT NULL,hex VARCHAR(254) NOT NULL);
+CREATE INDEX IF NOT EXISTS block_headers_hash_index on block_headers (hash);
+CREATE INDEX IF NOT EXISTS block_headers_height_index on block_headers (height);
+
+CREATE TABLE IF NOT EXISTS cfheaders (hash VARCHAR(254) PRIMARY KEY NOT NULL,filter_hash VARCHAR(254) NOT NULL,previous_filter_header VARCHAR(254) NOT NULL,block_hash VARCHAR(254) NOT NULL,height BIGINT NOT NULL);
+CREATE INDEX IF NOT EXISTS cfheaders_block_hash_index on cfheaders (block_hash);
+CREATE INDEX IF NOT EXISTS cfheaders_height_index on cfheaders (height);
+ALTER TABLE cfheaders ADD CONSTRAINT cfh_block_hash_fk FOREIGN KEY (block_hash) REFERENCES block_headers(hash);
+
+CREATE TABLE IF NOT EXISTS cfilters (hash VARCHAR(254) NOT NULL,filter_type INTEGER NOT NULL,bytes VARCHAR(254) NOT NULL,height BIGINT NOT NULL,block_hash VARCHAR(254) PRIMARY KEY NOT NULL);
+CREATE INDEX IF NOT EXISTS cfilters_hash_index on cfilters (hash);
+CREATE INDEX IF NOT EXISTS cfilters_height_index on cfilters (height);
+ALTER TABLE cfilters ADD CONSTRAINT cf_block_hash_fk FOREIGN KEY (block_hash) REFERENCES block_headers(hash);

chain/src/main/scala/org/bitcoins/chain/models/BlockHeaderDAO.scala

@@ -20,8 +20,9 @@ case class BlockHeaderDAO()(
     extends CRUD[BlockHeaderDb, DoubleSha256DigestBE]
     with SlickUtil[BlockHeaderDb, DoubleSha256DigestBE] {
 
-  import org.bitcoins.db.DbCommonsColumnMappers._
   import profile.api._
+  private val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
+  import mappers._
 
   override val table =
     profile.api.TableQuery[BlockHeaderTable]
@@ -290,7 +291,6 @@ case class BlockHeaderDAO()(
   /** A table that stores block headers related to a blockchain */
   class BlockHeaderTable(tag: Tag)
       extends Table[BlockHeaderDb](tag, "block_headers") {
-    import org.bitcoins.db.DbCommonsColumnMappers._
 
     def height = column[Int]("height")
 

chain/src/main/scala/org/bitcoins/chain/models/CompactFilterDAO.scala

@@ -13,12 +13,12 @@ case class CompactFilterDAO()(
     override val appConfig: ChainAppConfig)
     extends CRUD[CompactFilterDb, DoubleSha256DigestBE]
     with SlickUtil[CompactFilterDb, DoubleSha256DigestBE] {
-  import org.bitcoins.db.DbCommonsColumnMappers._
+  val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
+  import mappers._
   import profile.api._
 
   class CompactFilterTable(tag: Tag)
       extends Table[CompactFilterDb](tag, "cfilters") {
-    import org.bitcoins.db.DbCommonsColumnMappers._
 
     def hash = column[DoubleSha256DigestBE]("hash")
 

chain/src/main/scala/org/bitcoins/chain/models/CompactFilterHeaderDAO.scala

@@ -12,7 +12,8 @@ case class CompactFilterHeaderDAO()(
     extends CRUD[CompactFilterHeaderDb, DoubleSha256DigestBE]
     with SlickUtil[CompactFilterHeaderDb, DoubleSha256DigestBE] {
   import profile.api._
-  import org.bitcoins.db.DbCommonsColumnMappers._
+  val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
+  import mappers._
 
   class CompactFilterHeaderTable(tag: Tag)
       extends Table[CompactFilterHeaderDb](tag, "cfheaders") {

db-commons-test/src/test/scala/org/bitcoins/db/DbManagementTest.scala

@@ -5,7 +5,7 @@ import org.bitcoins.chain.config.ChainAppConfig
 import org.bitcoins.chain.db.ChainDbManagement
 import org.bitcoins.node.config.NodeAppConfig
 import org.bitcoins.node.db.NodeDbManagement
-import org.bitcoins.testkit.BitcoinSTestAppConfig
+import org.bitcoins.testkit.{BitcoinSTestAppConfig, EmbeddedPg}
 import org.bitcoins.testkit.BitcoinSTestAppConfig.ProjectType
 import org.bitcoins.testkit.util.{BitcoinSAsyncTest, BitcoinSUnitTest}
 import org.bitcoins.wallet.config.WalletAppConfig
@@ -13,10 +13,10 @@ import org.bitcoins.wallet.db.WalletDbManagement
 
 import scala.concurrent.ExecutionContext
 
-class DbManagementTest extends BitcoinSAsyncTest {
+class DbManagementTest extends BitcoinSAsyncTest with EmbeddedPg {
 
   def dbConfig(project: ProjectType): Config = {
-    BitcoinSTestAppConfig.configWithMemoryDb(Some(project))
+    BitcoinSTestAppConfig.configWithEmbeddedDb(Some(project), pgUrl)
   }
 
   def createChainDbManagement(
@@ -57,7 +57,8 @@ class DbManagementTest extends BitcoinSAsyncTest {
                                           dbConfig(ProjectType.Wallet))
     val walletDbManagement = createWalletDbManagement(walletAppConfig)
     val result = walletDbManagement.migrate()
-    assert(result == 4)
+    val expected = if (walletAppConfig.driverName == "postgresql") 1 else 4
+    assert(result == expected)
   }
 
   it must "run migrations for node db" in {

db-commons/src/main/resources/db.conf

@@ -7,6 +7,8 @@ common = {
     //https://scala-slick.org/doc/3.3.1/api/index.html#slick.jdbc.JdbcBackend$DatabaseFactoryDef@forConfig(String,Config,Driver,ClassLoader):Database
     path = ${bitcoin-s.datadir}/${bitcoin-s.network}/
     driver = org.sqlite.JDBC
+    username = ""
+    password = ""
 
     numThreads = 5 # default num threads is 20, which is way too much
     # as long as we're on SQLite there's no point
@@ -22,8 +24,14 @@ bitcoin-s {
     db {
       name = walletdb.sqlite
       url = "jdbc:sqlite:"${bitcoin-s.wallet.db.path}${bitcoin-s.wallet.db.name}
-
     }
+    # PostgreSQL example:
+    # db {
+    #   url = "jdbc:postgresql://localhost:5432/"${bitcoin-s.wallet.db.name}
+    #   driver = "org.postgresql.Driver"
+    #   username = postgres
+    #   password = ""
+    # }
   }
 
   node = ${common}

db-commons/src/main/scala/org/bitcoins/db/AppConfig.scala

@@ -1,26 +1,21 @@
 package org.bitcoins.db
 
-import org.bitcoins.core.config.NetworkParameters
-import org.bitcoins.core.protocol.blockchain.ChainParams
-import java.nio.file.Path
-import java.nio.file.Paths
+import java.nio.file.{Files, Path, Paths}
 
-import org.bitcoins.core.config.MainNet
-import org.bitcoins.core.config.TestNet3
-import org.bitcoins.core.config.RegTest
+import ch.qos.logback.classic.Level
 import com.typesafe.config._
+import org.bitcoins.core.config.{MainNet, NetworkParameters, RegTest, TestNet3}
+import org.bitcoins.core.protocol.blockchain.{
+  ChainParams,
+  MainNetChainParams,
+  RegTestNetChainParams,
+  TestNetChainParams
+}
 import org.bitcoins.core.util.BitcoinSLogger
 
-import org.bitcoins.core.protocol.blockchain.MainNetChainParams
-import org.bitcoins.core.protocol.blockchain.TestNetChainParams
-import org.bitcoins.core.protocol.blockchain.RegTestNetChainParams
-import java.nio.file.Files
-
+import scala.concurrent.{ExecutionContext, Future}
 import scala.util.Properties
 import scala.util.matching.Regex
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-import ch.qos.logback.classic.Level
 
 /**
   * Everything needed to configure functionality
@@ -98,7 +93,7 @@ abstract class AppConfig extends LoggerConfig {
       // that as our config. here we have to do the reverse, to
       // get the keys to resolve correctly
       val reconstructedStr = s"""
-      bitcoin-s: ${this.config.asReadableJson}
+      "bitcoin-s": ${this.config.asReadableJson}
       """
       val reconstructed = ConfigFactory.parseString(reconstructedStr)
       newConfigOfType(reconstructed +: configOverrides)

db-commons/src/main/scala/org/bitcoins/db/CRUD.scala

@@ -173,6 +173,7 @@ case class SafeDatabase(jdbcProfile: JdbcProfileComponent[AppConfig])
     * the database.
     */
   private val foreignKeysPragma = sqlu"PRAGMA foreign_keys = TRUE;"
+  private val sqlite = jdbcProfile.driverName == "sqlite"
 
   /** Logs the given action and error, if we are not on mainnet */
   private def logAndThrowError(
@@ -187,7 +188,9 @@ case class SafeDatabase(jdbcProfile: JdbcProfileComponent[AppConfig])
   /** Runs the given DB action */
   def run[R](action: DBIOAction[R, NoStream, _])(
       implicit ec: ExecutionContext): Future[R] = {
-    val result = database.run[R](foreignKeysPragma >> action)
+    val result =
+      if (sqlite) database.run[R](foreignKeysPragma >> action)
+      else database.run[R](action)
     result.recoverWith { logAndThrowError(action) }
   }
 
@@ -197,7 +200,9 @@ case class SafeDatabase(jdbcProfile: JdbcProfileComponent[AppConfig])
     */
   def runVec[R](action: DBIOAction[Seq[R], NoStream, _])(
       implicit ec: ExecutionContext): Future[Vector[R]] = {
-    val result = database.run[Seq[R]](foreignKeysPragma >> action)
+    val result =
+      if (sqlite) database.run[Seq[R]](foreignKeysPragma >> action)
+      else database.run[Seq[R]](action)
     result.map(_.toVector).recoverWith { logAndThrowError(action) }
   }
 }

db-commons/src/main/scala/org/bitcoins/db/DbCommonsColumnMappers.scala

@@ -22,10 +22,11 @@ import org.bitcoins.crypto.{
   Sha256Hash160Digest
 }
 import scodec.bits.ByteVector
-import slick.jdbc.GetResult
-import slick.jdbc.SQLiteProfile.api._
+import slick.jdbc.{GetResult, JdbcProfile}
 
-abstract class DbCommonsColumnMappers {
+class DbCommonsColumnMappers(val profile: JdbcProfile) {
+
+  import profile.api._
 
   /**
     * If executing something like this:
@@ -178,5 +179,3 @@ abstract class DbCommonsColumnMappers {
       .base[SatoshisPerByte, Long](_.toLong, SatoshisPerByte.fromLong)
   }
 }
-
-object DbCommonsColumnMappers extends DbCommonsColumnMappers

db-commons/src/main/scala/org/bitcoins/db/DbManagement.scala

@@ -1,6 +1,6 @@
 package org.bitcoins.db
 
-import org.bitcoins.core.util.BitcoinSLogger
+import org.bitcoins.core.util.{BitcoinSLogger, FutureUtil}
 import org.flywaydb.core.Flyway
 import org.flywaydb.core.api.FlywayException
 
@@ -33,26 +33,16 @@ trait DbManagement extends BitcoinSLogger {
 
   def allTables: List[TableQuery[Table[_]]]
 
-  /** Creates all tables in our table list, in one SQL transaction */
-  def createAll()(implicit ec: ExecutionContext): Future[Unit] = {
-    val query = {
-      val querySeq =
-        allTables
-          .map(createTableQuery(_, createIfNotExists = true))
-          .map { query =>
-            // DIRTY HACK. For some reason Slick doesn't know that Sqlite can do CREATE INDEX IF NOT EXISTS
-            val statements = query.statements.map(
-              _.replace("create index", "create index if not exists"))
-            query.overrideStatements(statements)
-          }
-      DBIO.seq(querySeq: _*).transactionally
-    }
-
-    database.run(query).map(_ => logger.debug(s"Created tables"))
-  }
-
   def dropAll()(implicit ec: ExecutionContext): Future[Unit] = {
-    Future.sequence(allTables.reverse.map(dropTable(_))).map(_ => ())
+    val result =
+      FutureUtil
+        .foldLeftAsync((), allTables.reverse) { (_, table) =>
+          dropTable(table)
+        }
+    result.failed.foreach { e =>
+      e.printStackTrace()
+    }
+    result
   }
 
   /** The query needed to create the given table */
@@ -86,18 +76,25 @@ trait DbManagement extends BitcoinSLogger {
     result
   }
 
+  def dropTable(tableName: String): Future[Int] = {
+    val result = database.run(sqlu"""DROP TABLE IF EXISTS #$tableName""")
+    import scala.concurrent.ExecutionContext.Implicits.global
+    result.failed.foreach { ex =>
+      ex.printStackTrace()
+    }
+    result
+  }
+
   /** Executes migrations related to this database
     *
     * @see [[https://flywaydb.org/documentation/api/#programmatic-configuration-java]] */
   def migrate(): Int = {
-    val url = jdbcUrl
-    val username = ""
-    val password = ""
-    //appConfig.dbName is for the format 'walletdb.sqlite' or 'nodedb.sqlite' etc
-    //we need to remove the '.sqlite' suffix
-    val name = dbName.split('.').head.mkString
-    val config = Flyway.configure().locations(s"classpath:${name}/migration/")
-    val flyway = config.dataSource(url, username, password).load
+    val module = appConfig.moduleName
+    val config =
+      Flyway
+        .configure()
+        .locations(s"classpath:${driverName}/${module}/migration/")
+    val flyway = config.dataSource(jdbcUrl, username, password).load
 
     try {
       flyway.migrate()

db-commons/src/main/scala/org/bitcoins/db/JdbcProfileComponent.scala

@@ -45,6 +45,17 @@ trait JdbcProfileComponent[+ConfigType <: AppConfig] extends BitcoinSLogger {
     dbConfig.config.getString("db.url")
   }
 
+  lazy val driverName: String = {
+    val parts = jdbcUrl.split(":")
+    require(parts.size >= 2 && parts(0) == "jdbc",
+            s"`${jdbcUrl}` must be a valid JDBC URL")
+    parts(1)
+  }
+
+  lazy val username: String = dbConfig.config.getString("db.username")
+
+  lazy val password: String = dbConfig.config.getString("db.password")
+
   /** The database we are connecting to */
   lazy val database: Database = {
     dbConfig.db

docs/config/configuration.md

@@ -195,4 +195,55 @@ akka {
         }
     }
 }
+```
+
+Sqlite and PostgreSQL
+
+By default, bitcoin-s uses Sqlite to store its data. 
+It creates three Sqlite databases in `~/.bitcoin-s/${network}`: `chain.sqlite` for `chain` project, 
+`node.sqlite` for `node` project and `wallet.sqlite` the wallet. This is the default configuration, 
+it doesn't require additional changes in the config file. 
+
+`bitcoin-s` also supports PostgreSQL as a database backend. In order to use a 
+PostgreSQL database for all project you need to add following into your config file: 
+
+```$xslt
+bitcoin-s {
+    common {
+        profile = "slick.jdbc.PostgresProfile$"
+        db {
+            url = "jdbc:postgresql://localhost:5432/database"
+            driver = "org.postgresql.Driver"
+            username = "user"
+            password = "topsecret"
+        }
+    }
+}
+```
+
+Also you can use mix databases and drivers in one configuration. For example,
+This configuration file enables Sqlite for `node` project (it's default, so its configuration 
+is omitted), and `walletdb` and `chaindb` PostgreSQL databases for `wallet` and `chain` projects:
+
+```$xslt
+bitcoin-s {
+    chain {
+        profile = "slick.jdbc.PostgresProfile$"
+        db {
+            url = "jdbc:postgresql://localhost:5432/chaindb"
+            driver = "org.postgresql.Driver"
+            username = "user"
+            password = "topsecret"
+        }
+    }
+    wallet {
+        profile = "slick.jdbc.PostgresProfile$"
+        db {
+            url = "jdbc:postgresql://localhost:5432/walletdb"
+            driver = "org.postgresql.Driver"
+            username = "user"
+            password = "topsecret"
+        }
+    }
+}
 ```

node-test/src/test/resources/logback-test.xml

@@ -0,0 +1,9 @@
+<configuration>
+    <include resource="common-logback.xml" />
+
+    <root level="OFF">
+        <appender-ref ref="STDOUT" />
+        <appender-ref ref="FILE"/>
+    </root>
+
+</configuration>

node-test/src/test/scala/org/bitcoins/node/BroadcastTransactionTest.scala

@@ -7,7 +7,10 @@ import org.bitcoins.rpc.BitcoindException
 import org.bitcoins.server.BitcoinSAppConfig
 import org.bitcoins.testkit.BitcoinSTestAppConfig
 import org.bitcoins.testkit.async.TestAsyncUtil
-import org.bitcoins.testkit.node.NodeUnitTest.NeutrinoNodeFundedWalletBitcoind
+import org.bitcoins.testkit.node.NodeUnitTest.{
+  NeutrinoNodeFundedWalletBitcoind,
+  SpvNodeFundedWalletBitcoind
+}
 import org.bitcoins.testkit.node.{NodeTestUtil, NodeUnitTest}
 import org.scalatest.FutureOutcome
 
@@ -18,15 +21,15 @@ class BroadcastTransactionTest extends NodeUnitTest {
 
   /** Wallet config with data directory set to user temp directory */
   implicit override protected def config: BitcoinSAppConfig =
-    BitcoinSTestAppConfig.getNeutrinoTestConfig()
+    BitcoinSTestAppConfig.getSpvWithEmbeddedDbTestConfig(pgUrl)
 
-  override type FixtureParam = NeutrinoNodeFundedWalletBitcoind
+  override type FixtureParam = SpvNodeFundedWalletBitcoind
 
   def withFixture(test: OneArgAsyncTest): FutureOutcome =
-    withNeutrinoNodeFundedWalletBitcoind(test, NodeCallbacks.empty)
+    withSpvNodeFundedWalletBitcoind(test, NodeCallbacks.empty)
 
   it must "broadcast a transaction" in { param =>
-    val NeutrinoNodeFundedWalletBitcoind(node, wallet, rpc) = param
+    val SpvNodeFundedWalletBitcoind(node, wallet, rpc) = param
 
     def hasSeenTx(transaction: Transaction): Future[Boolean] = {
       rpc

node-test/src/test/scala/org/bitcoins/node/NeutrinoNodeTest.scala

@@ -20,7 +20,7 @@ class NeutrinoNodeTest extends NodeUnitTest {
 
   /** Wallet config with data directory set to user temp directory */
   implicit override protected def config: BitcoinSAppConfig =
-    BitcoinSTestAppConfig.getNeutrinoTestConfig()
+    BitcoinSTestAppConfig.getNeutrinoWithEmbeddedDbTestConfig(pgUrl)
 
   override type FixtureParam = NeutrinoNodeFundedWalletBitcoind
 

node-test/src/test/scala/org/bitcoins/node/NeutrinoNodeWithWalletTest.scala

@@ -22,7 +22,7 @@ class NeutrinoNodeWithWalletTest extends NodeUnitTest {
 
   /** Wallet config with data directory set to user temp directory */
   implicit override protected def config: BitcoinSAppConfig =
-    BitcoinSTestAppConfig.getNeutrinoTestConfig()
+    BitcoinSTestAppConfig.getNeutrinoWithEmbeddedDbTestConfig(pgUrl)
 
   override type FixtureParam = NeutrinoNodeFundedWalletBitcoind
 

node-test/src/test/scala/org/bitcoins/node/SpvNodeTest.scala

@@ -15,7 +15,7 @@ class SpvNodeTest extends NodeUnitTest {
 
   /** Wallet config with data directory set to user temp directory */
   implicit override protected def config: BitcoinSAppConfig =
-    BitcoinSTestAppConfig.getSpvTestConfig()
+    BitcoinSTestAppConfig.getSpvWithEmbeddedDbTestConfig(pgUrl)
 
   override type FixtureParam = SpvNodeConnectedWithBitcoind
 

node-test/src/test/scala/org/bitcoins/node/SpvNodeWithWalletTest.scala

@@ -19,7 +19,7 @@ class SpvNodeWithWalletTest extends NodeUnitTest {
 
   /** Wallet config with data directory set to user temp directory */
   implicit override protected def config: BitcoinSAppConfig =
-    BitcoinSTestAppConfig.getSpvTestConfig()
+    BitcoinSTestAppConfig.getSpvWithEmbeddedDbTestConfig(pgUrl)
 
   override type FixtureParam = SpvNodeFundedWalletBitcoind
 

node-test/src/test/scala/org/bitcoins/node/UpdateBloomFilterTest.scala

@@ -22,7 +22,7 @@ class UpdateBloomFilterTest extends NodeUnitTest with BeforeAndAfter {
 
   /** Wallet config with data directory set to user temp directory */
   implicit override protected def config: BitcoinSAppConfig =
-    BitcoinSTestAppConfig.getSpvTestConfig()
+    BitcoinSTestAppConfig.getSpvWithEmbeddedDbTestConfig(pgUrl)
 
   override type FixtureParam = SpvNodeFundedWalletBitcoind
 
@@ -90,7 +90,6 @@ class UpdateBloomFilterTest extends NodeUnitTest with BeforeAndAfter {
 
     for {
       firstBloom <- wallet.getBloomFilter()
-
       addressFromBitcoind <- rpc.getNewAddress
       tx <- wallet
         .sendToAddress(addressFromBitcoind,

node-test/src/test/scala/org/bitcoins/node/models/BroadcastAbleTransactionDAOTest.scala

@@ -1,16 +1,16 @@
 package org.bitcoins.node.models
 
 import org.bitcoins.server.BitcoinSAppConfig
-import org.bitcoins.testkit.BitcoinSTestAppConfig
+import org.bitcoins.testkit.{BitcoinSTestAppConfig, EmbeddedPg}
 import org.bitcoins.testkit.Implicits._
 import org.bitcoins.testkit.core.gen.TransactionGenerators
 import org.bitcoins.testkit.fixtures.NodeDAOFixture
 
-class BroadcastAbleTransactionDAOTest extends NodeDAOFixture {
+class BroadcastAbleTransactionDAOTest extends NodeDAOFixture with EmbeddedPg {
 
   /** Wallet config with data directory set to user temp directory */
   implicit override protected def config: BitcoinSAppConfig =
-    BitcoinSTestAppConfig.getSpvTestConfig()
+    BitcoinSTestAppConfig.getSpvWithEmbeddedDbTestConfig(pgUrl)
 
   behavior of "BroadcastAbleTransactionDAO"
 

node-test/src/test/scala/org/bitcoins/node/networking/P2PClientTest.scala

@@ -118,14 +118,13 @@ class P2PClientTest extends BitcoindRpcTest {
 
   override def beforeAll(): Unit = {
     implicit val chainConf = config.chainConf
-    for {
-      _ <- chainConf.createAll()
-    } yield ()
+    chainConf.migrate()
   }
 
   override def afterAll(): Unit = {
     implicit val chainConf = config.chainConf
     for {
+      _ <- chainConf.dropTable("flyway_schema_history")
       _ <- chainConf.dropAll()
     } yield ()
     super.afterAll()

node/src/main/resources/postgresql/node/migration/V1__node_db_baseline.sql

@@ -0,0 +1 @@
+CREATE TABLE IF NOT EXISTS broadcast_elements (txid TEXT NOT NULL UNIQUE,tx_bytes TEXT NOT NULL,id SERIAL PRIMARY KEY NOT NULL);

node/src/main/scala/org/bitcoins/node/models/BroadcastAbleTransactionDAO.scala

@@ -17,6 +17,8 @@ final case class BroadcastAbleTransactionDAO()(
     extends CRUDAutoInc[BroadcastAbleTransaction] {
 
   import profile.api._
+  val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
+  import mappers._
 
   override val table: profile.api.TableQuery[BroadcastAbleTransactionTable] =
     profile.api.TableQuery[BroadcastAbleTransactionTable]
@@ -24,7 +26,8 @@ final case class BroadcastAbleTransactionDAO()(
   /** Searches for a TX by its TXID */
   def findByHash(
       hash: DoubleSha256Digest): Future[Option[BroadcastAbleTransaction]] = {
-    import org.bitcoins.db.DbCommonsColumnMappers._
+    val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
+    import mappers._
 
     val query = table.filter(_.txid === hash.flip)
     database.run(query.result).map(_.headOption)
@@ -45,8 +48,6 @@ final case class BroadcastAbleTransactionDAO()(
     private val toTuple: BroadcastAbleTransaction => Option[Tuple] = tx =>
       Some(tx.transaction.txId.flip, tx.transaction.bytes, tx.id)
 
-    import org.bitcoins.db.DbCommonsColumnMappers._
-
     def txid: Rep[DoubleSha256DigestBE] = column("txid", O.Unique)
     def bytes: Rep[ByteVector] = column("tx_bytes")
 

project/Deps.scala

@@ -26,12 +26,13 @@ object Deps {
     val asyncNewScalaV = "0.10.0"
 
     val flywayV = "6.4.2"
-    val postgresV = "9.4.1210"
+    val postgresV = "42.2.12"
     val akkaActorV = akkaStreamv
     val slickV = "3.3.2"
     val sqliteV = "3.31.1"
     val scalameterV = "0.17"
     val scalamockV = "4.4.0"
+    val pgEmbeddedV = "0.13.3"
 
     val newMicroPickleV = "0.8.0"
     val newMicroJsonV = newMicroPickleV
@@ -117,6 +118,7 @@ object Deps {
     val scalacheck = "org.scalacheck" %% "scalacheck" % V.scalacheck withSources () withJavadoc ()
     val scalaTest = "org.scalatest" %% "scalatest" % V.scalaTest withSources () withJavadoc ()
     val scalaTestPlus = "org.scalatestplus" %% "scalacheck-1-14" % V.scalaTestPlus withSources () withJavadoc ()
+    val pgEmbedded = "com.opentable.components" % "otj-pg-embedded" % V.pgEmbeddedV withSources () withJavadoc ()
   }
 
   object Test {
@@ -132,13 +134,16 @@ object Deps {
     val playJson = Compile.playJson % "test"
     val akkaTestkit = "com.typesafe.akka" %% "akka-testkit" % V.akkaActorV withSources () withJavadoc ()
     val scalameter = "com.storm-enroute" %% "scalameter" % V.scalameterV % "test" withSources () withJavadoc ()
+    val pgEmbedded = "com.opentable.components" % "otj-pg-embedded" % V.pgEmbeddedV % "test" withSources () withJavadoc ()
   }
 
   val chain = List(
     Compile.logback
   )
 
-  val chainTest = List()
+  val chainTest = List(
+    Test.pgEmbedded
+  )
 
   def appCommons(scalaVersion: String) = List(
     Compile.newMicroPickle,
@@ -208,8 +213,10 @@ object Deps {
     Compile.sourcecode,
     Compile.logback,
     Compile.sqlite,
+    Compile.postgres,
     Compile.slickHikari,
-    Test.scalaTest
+    Test.scalaTest,
+    Test.pgEmbedded
   )
 
   def cli(scalaVersion: String) = List(
@@ -268,7 +275,8 @@ object Deps {
 
   val nodeTest = List(
     Test.akkaTestkit,
-    Test.scalaTest
+    Test.scalaTest,
+    Test.pgEmbedded
   )
 
   val testkit = List(
@@ -276,6 +284,7 @@ object Deps {
     Compile.scalacheck,
     Compile.scalaTest,
     Compile.scalaTestPlus,
+    Compile.pgEmbedded,
     Test.akkaTestkit
   )
 
@@ -294,7 +303,8 @@ object Deps {
   )
 
   val walletTest = List(
-    Test.akkaTestkit
+    Test.akkaTestkit,
+    Test.pgEmbedded
   )
 
   val docs = List(

testkit/src/main/scala/org/bitcoins/testkit/BitcoinSTestAppConfig.scala

@@ -33,6 +33,25 @@ object BitcoinSTestAppConfig {
     BitcoinSAppConfig(tmpDir(), (overrideConf +: config): _*)
   }
 
+  def getSpvWithEmbeddedDbTestConfig(
+      pgUrl: ProjectType => Option[String],
+      config: Config*)(implicit ec: ExecutionContext): BitcoinSAppConfig = {
+    val overrideConf = ConfigFactory.parseString {
+      """
+        |bitcoin-s {
+        |  node {
+        |     mode = spv
+        |  }
+        |}
+      """.stripMargin
+    }
+
+    BitcoinSAppConfig(tmpDir(),
+                      (overrideConf +: configWithEmbeddedDb(
+                        project = None,
+                        pgUrl) +: config): _*)
+  }
+
   def getNeutrinoTestConfig(config: Config*)(
       implicit ec: ExecutionContext): BitcoinSAppConfig = {
     val overrideConf = ConfigFactory.parseString {
@@ -47,6 +66,24 @@ object BitcoinSTestAppConfig {
     BitcoinSAppConfig(tmpDir(), (overrideConf +: config): _*)
   }
 
+  def getNeutrinoWithEmbeddedDbTestConfig(
+      pgUrl: ProjectType => Option[String],
+      config: Config*)(implicit ec: ExecutionContext): BitcoinSAppConfig = {
+    val overrideConf = ConfigFactory.parseString {
+      """
+        |bitcoin-s {
+        |  node {
+        |     mode = neutrino
+        |  }
+        |}
+      """.stripMargin
+    }
+    BitcoinSAppConfig(tmpDir(),
+                      (overrideConf +: configWithEmbeddedDb(
+                        project = None,
+                        pgUrl) +: config): _*)
+  }
+
   sealed trait ProjectType
 
   object ProjectType {
@@ -62,21 +99,33 @@ object BitcoinSTestAppConfig {
     * project is given). This configuration can then be
     * given as a override to other configs.
     */
-  def configWithMemoryDb(project: Option[ProjectType]): Config = {
-    def memConfigForProject(project: ProjectType): String = {
+  def configWithEmbeddedDb(
+      project: Option[ProjectType],
+      pgUrl: ProjectType => Option[String]): Config = {
+
+    def pgConfigForProject(project: ProjectType): String = {
       val name = project.toString().toLowerCase()
-      s"""
+      s""" $name.profile = "slick.jdbc.PostgresProfile$$"
          | $name.db {
-         |   url = "jdbc:sqlite:file:$name.db:?mode=memory&cache=shared"
+         |   url = "${pgUrl(project).getOrElse(
+           throw new RuntimeException(s"Cannot get db url for $project"))}"
+         |   driver = "org.postgresql.Driver"
+         |   username = "postgres"
+         |   password = ""
          |   connectionPool = disabled
          |   keepAliveConnection = true
-         | }
-         |""".stripMargin
+         | }""".stripMargin
     }
 
+    def configForProject(project: ProjectType) =
+      if (pgUrl(project).isDefined)
+        pgConfigForProject(project)
+      else
+        ""
+
     val confStr = project match {
-      case None    => ProjectType.all.map(memConfigForProject).mkString("\n")
-      case Some(p) => memConfigForProject(p)
+      case None    => ProjectType.all.map(configForProject).mkString("\n")
+      case Some(p) => configForProject(p)
     }
     val nestedConfStr = s"""
                            | bitcoin-s {

testkit/src/main/scala/org/bitcoins/testkit/EmbeddedPg.scala

@@ -0,0 +1,52 @@
+package org.bitcoins.testkit
+
+import com.opentable.db.postgres.embedded.EmbeddedPostgres
+import org.bitcoins.testkit.BitcoinSTestAppConfig.ProjectType
+import org.scalatest.{BeforeAndAfterAll, Suite}
+
+import scala.util.Try
+
+trait EmbeddedPg extends BeforeAndAfterAll { this: Suite =>
+
+  val pgEnabled: Boolean = sys.env.contains("PG_ENABLED")
+
+  val pg: Option[EmbeddedPostgres] =
+    if (pgEnabled) Some(EmbeddedPostgres.start()) else None
+
+  def pgUrl(dbname: String): Option[String] =
+    pg.map(_.getJdbcUrl("postgres", dbname))
+
+  def pgUrl(project: ProjectType): Option[String] = project match {
+    case ProjectType.Wallet => pgUrl("walletdb")
+    case ProjectType.Node   => pgUrl("nodedb")
+    case ProjectType.Chain  => pgUrl("chaindb")
+  }
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    executePgSql(s"CREATE DATABASE chaindb")
+    executePgSql(s"CREATE DATABASE walletdb")
+    executePgSql(s"CREATE DATABASE nodedb")
+  }
+
+  override def afterAll(): Unit = {
+    super.afterAll()
+    Try(executePgSql(s"DROP DATABASE nodedb"))
+    Try(executePgSql(s"DROP DATABASE walletdb"))
+    Try(executePgSql(s"DROP DATABASE chaindb"))
+    Try(pg.foreach(_.close()))
+    ()
+  }
+
+  def executePgSql(sql: String): Unit = pg.foreach { pg =>
+    val conn = pg.getPostgresDatabase.getConnection
+    try {
+      val st = conn.createStatement()
+      try {
+        st.execute(sql)
+      } finally st.close()
+
+    } finally conn.close()
+  }
+
+}

testkit/src/main/scala/org/bitcoins/testkit/chain/ChainDbUnitTest.scala

@@ -0,0 +1,27 @@
+package org.bitcoins.testkit.chain
+
+import com.typesafe.config.ConfigFactory
+import org.bitcoins.chain.config.ChainAppConfig
+import org.bitcoins.testkit.BitcoinSTestAppConfig.ProjectType
+import org.bitcoins.testkit.{BitcoinSTestAppConfig, EmbeddedPg}
+
+trait ChainDbUnitTest extends ChainUnitTest with EmbeddedPg {
+
+  implicit override lazy val appConfig: ChainAppConfig = {
+    val memoryDb =
+      BitcoinSTestAppConfig.configWithEmbeddedDb(Some(ProjectType.Chain), pgUrl)
+    val chainConfig: ChainAppConfig =
+      BitcoinSTestAppConfig.getSpvTestConfig()
+    chainConfig.withOverrides(memoryDb)
+  }
+
+  override lazy val mainnetAppConfig: ChainAppConfig = {
+    val memoryDb =
+      BitcoinSTestAppConfig.configWithEmbeddedDb(Some(ProjectType.Chain), pgUrl)
+    val mainnetConf = ConfigFactory.parseString("bitcoin-s.network = mainnet")
+    val chainConfig: ChainAppConfig =
+      BitcoinSTestAppConfig.getSpvTestConfig(mainnetConf)
+    chainConfig.withOverrides(memoryDb)
+  }
+
+}

testkit/src/main/scala/org/bitcoins/testkit/chain/ChainUnitTest.scala

@@ -461,15 +461,18 @@ object ChainUnitTest extends ChainVerificationLogger {
 
   def setupAllTables()(
       implicit appConfig: ChainAppConfig,
-      ec: ExecutionContext): Future[Unit] = {
-    appConfig.createAll()
+      ec: ExecutionContext): Future[Unit] = Future {
+    appConfig.migrate()
+    ()
   }
 
   def destroyAllTables()(
       implicit appConfig: ChainAppConfig,
-      ec: ExecutionContext): Future[Unit] = {
-    appConfig.dropAll()
-  }
+      ec: ExecutionContext): Future[Unit] =
+    for {
+      _ <- appConfig.dropTable("flyway_schema_history")
+      _ <- appConfig.dropAll()
+    } yield ()
 
   /** Creates the [[org.bitcoins.chain.models.BlockHeaderTable]] and inserts the genesis header */
   def setupHeaderTableWithGenesisHeader()(

testkit/src/main/scala/org/bitcoins/testkit/fixtures/NodeDAOFixture.scala

@@ -4,6 +4,8 @@ import org.bitcoins.node.models.BroadcastAbleTransactionDAO
 import org.bitcoins.testkit.node.NodeUnitTest
 import org.scalatest._
 
+import scala.concurrent.Future
+
 case class NodeDAOs(txDAO: BroadcastAbleTransactionDAO)
 
 /** Provides a fixture where all DAOs used by the node projects are provided */
@@ -17,8 +19,9 @@ trait NodeDAOFixture extends NodeUnitTest {
 
   def withFixture(test: OneArgAsyncTest): FutureOutcome =
     makeFixture(build = () =>
-                  nodeConfig
-                    .createAll()(executionContext)
+                  Future(
+                    nodeConfig
+                      .migrate())(executionContext)
                     .map(_ => daos),
                 destroy = () =>
                   nodeConfig

testkit/src/main/scala/org/bitcoins/testkit/fixtures/WalletDAOFixture.scala

@@ -6,6 +6,8 @@ import org.bitcoins.wallet.models._
 import org.scalatest._
 import org.scalatest.flatspec.FixtureAsyncFlatSpec
 
+import scala.concurrent.Future
+
 case class WalletDAOs(
     accountDAO: AccountDAO,
     addressDAO: AddressDAO,
@@ -31,6 +33,12 @@ trait WalletDAOFixture extends FixtureAsyncFlatSpec with BitcoinSWalletTest {
   implicit private val walletConfig: WalletAppConfig = config
 
   def withFixture(test: OneArgAsyncTest): FutureOutcome =
-    makeFixture(build = () => walletConfig.createAll().map(_ => daos),
-                destroy = () => walletConfig.dropAll())(test)
+    makeFixture(build = () => Future(walletConfig.migrate()).map(_ => daos),
+                destroy = () => dropAll())(test)
+
+  def dropAll(): Future[Unit] =
+    for {
+      _ <- walletConfig.dropTable("flyway_schema_history")
+      _ <- walletConfig.dropAll()
+    } yield ()
 }

testkit/src/main/scala/org/bitcoins/testkit/node/NodeUnitTest.scala

@@ -20,6 +20,7 @@ import org.bitcoins.rpc.client.common.BitcoindVersion.V18
 import org.bitcoins.rpc.client.common.{BitcoindRpcClient, BitcoindVersion}
 import org.bitcoins.server.BitcoinSAppConfig
 import org.bitcoins.server.BitcoinSAppConfig._
+import org.bitcoins.testkit.EmbeddedPg
 import org.bitcoins.testkit.chain.ChainUnitTest
 import org.bitcoins.testkit.fixtures.BitcoinSFixture
 import org.bitcoins.testkit.node.NodeUnitTest.NodeFundedWalletBitcoind
@@ -36,10 +37,15 @@ import org.scalatest.FutureOutcome
 import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
 
-trait NodeUnitTest extends BitcoinSFixture {
+trait NodeUnitTest extends BitcoinSFixture with EmbeddedPg {
 
   override def beforeAll(): Unit = {
     AppConfig.throwIfDefaultDatadir(config.nodeConf)
+    super[EmbeddedPg].beforeAll()
+  }
+
+  override def afterAll(): Unit = {
+    super[EmbeddedPg].afterAll()
   }
 
   /** Wallet config with data directory set to user temp directory */

testkit/src/main/scala/org/bitcoins/testkit/wallet/BitcoinSWalletTest.scala

@@ -18,7 +18,7 @@ import org.bitcoins.rpc.client.common.{BitcoindRpcClient, BitcoindVersion}
 import org.bitcoins.rpc.client.v19.BitcoindV19RpcClient
 import org.bitcoins.server.BitcoinSAppConfig
 import org.bitcoins.server.BitcoinSAppConfig._
-import org.bitcoins.testkit.BitcoinSTestAppConfig
+import org.bitcoins.testkit.{BitcoinSTestAppConfig, EmbeddedPg}
 import org.bitcoins.testkit.chain.SyncUtil
 import org.bitcoins.testkit.fixtures.BitcoinSFixture
 import org.bitcoins.testkit.keymanager.KeyManagerTestUtil
@@ -36,12 +36,15 @@ import scala.concurrent.{
   Promise
 }
 
-trait BitcoinSWalletTest extends BitcoinSFixture with WalletLogger {
+trait BitcoinSWalletTest
+    extends BitcoinSFixture
+    with WalletLogger
+    with EmbeddedPg {
   import BitcoinSWalletTest._
 
   /** Wallet config with data directory set to user temp directory */
   implicit protected def config: BitcoinSAppConfig =
-    BitcoinSTestAppConfig.getSpvTestConfig()
+    BitcoinSTestAppConfig.getSpvWithEmbeddedDbTestConfig(pgUrl)
 
   implicit protected def walletAppConfig: WalletAppConfig = {
     config.walletConf
@@ -49,6 +52,7 @@ trait BitcoinSWalletTest extends BitcoinSFixture with WalletLogger {
 
   override def beforeAll(): Unit = {
     AppConfig.throwIfDefaultDatadir(config.walletConf)
+    super[EmbeddedPg].beforeAll()
   }
 
   def nodeApi: NodeApi = MockNodeApi
@@ -591,11 +595,11 @@ object BitcoinSWalletTest extends WalletLogger {
 
   def destroyWallet(wallet: WalletApi): Future[Unit] = {
     import wallet.walletConfig.ec
-    val destroyWalletF =
-      wallet.walletConfig
-        .dropAll()
-        .map(_ => ())
-    destroyWalletF
+    for {
+
+      _ <- wallet.walletConfig.dropTable("flyway_schema_history")
+      _ <- wallet.walletConfig.dropAll()
+    } yield ()
   }
 
 }

wallet-test/src/test/scala/org/bitcoins/wallet/WalletUnitTest.scala

@@ -9,6 +9,7 @@ import org.bitcoins.core.util.FutureUtil
 import org.bitcoins.crypto.AesPassword
 import org.bitcoins.keymanager.KeyManagerUnlockError.MnemonicNotFound
 import org.bitcoins.keymanager.{KeyManagerUnlockError, WalletStorage}
+import org.bitcoins.testkit.EmbeddedPg
 import org.bitcoins.testkit.wallet.BitcoinSWalletTest
 import org.bitcoins.wallet.api.WalletApi.BlockMatchingResponse
 import org.bitcoins.wallet.api.WalletApi

wallet/src/main/resources/postgresql/wallet/migration/V4__wallet_db_pg_baseline.sql

@@ -0,0 +1,11 @@
+CREATE TABLE wallet_accounts (hd_purpose INTEGER NOT NULL,xpub TEXT NOT NULL,coin INTEGER NOT NULL,account_index INTEGER NOT NULL,constraint pk_account primary key(hd_purpose,coin,account_index));
+
+CREATE TABLE addresses (hd_purpose INTEGER NOT NULL,account_index INTEGER NOT NULL,hd_coin INTEGER NOT NULL,hd_chain_type INTEGER NOT NULL,address TEXT PRIMARY KEY NOT NULL,script_witness TEXT,script_pub_key TEXT NOT NULL UNIQUE,address_index INTEGER NOT NULL,pubkey TEXT NOT NULL,hashed_pubkey TEXT NOT NULL,script_type TEXT NOT NULL,constraint fk_account foreign key (hd_purpose,hd_coin,account_index) references wallet_accounts(hd_purpose,coin,account_index));
+
+CREATE TABLE tx_table (id SERIAL UNIQUE, "txIdBE" TEXT NOT NULL,transaction TEXT NOT NULL,"unsignedTxIdBE" TEXT NOT NULL,"unsignedTx" TEXT NOT NULL,"wTxIdBE" TEXT,"totalOutput" BIGINT NOT NULL,"numInputs" INTEGER NOT NULL,"numOutputs" INTEGER NOT NULL,locktime BIGINT NOT NULL, constraint pk_tx primary key ("txIdBE"));
+
+CREATE TABLE wallet_incoming_txs (id SERIAL UNIQUE,"txIdBE" TEXT NOT NULL,"incomingAmount" BIGINT NOT NULL,constraint fk_underlying_tx foreign key("txIdBE") references tx_table("txIdBE") on update NO ACTION on delete NO ACTION, constraint pk_in_tx primary key ("txIdBE"));
+
+CREATE TABLE wallet_outgoing_txs (id SERIAL UNIQUE,"txIdBE" TEXT NOT NULL,"inputAmount" BIGINT NOT NULL,"sentAmount" BIGINT NOT NULL,"actualFee" BIGINT NOT NULL,"expectedFee" BIGINT NOT NULL,"feeRate" BIGINT NOT NULL,constraint fk_underlying_tx foreign key("txIdBE") references tx_table("txIdBE") on update NO ACTION on delete NO ACTION, constraint pk_out_tx primary key ("txIdBE"));
+
+CREATE TABLE txo_spending_info (id SERIAL PRIMARY KEY,tx_outpoint TEXT NOT NULL, script_pub_key TEXT NOT NULL,value BIGINT NOT NULL,hd_privkey_path TEXT NOT NULL,redeem_script TEXT,script_witness TEXT,txid TEXT NOT NULL,block_hash TEXT, txo_state TEXT NOT NULL, constraint fk_scriptPubKey foreign key(script_pub_key) references addresses(script_pub_key), constraint fk_incoming_txId foreign key(txid) references wallet_incoming_txs("txIdBE") on update NO ACTION on delete NO ACTION);

wallet/src/main/scala/org/bitcoins/wallet/Wallet.scala

@@ -497,10 +497,12 @@ object Wallet extends WalletLogger {
     val accountCreationF =
       createAccountFutures.flatMap(accounts => Future.sequence(accounts))
 
-    accountCreationF.foreach(_ =>
-      logger.debug(s"Created root level accounts for wallet"))
+    accountCreationF.foreach { _ =>
+      logger.debug(s"Created root level accounts for wallet")
+    }
 
     accountCreationF.failed.foreach { err =>
+      err.printStackTrace()
       logger.error(s"Failed to create root level accounts: $err")
     }
 

wallet/src/main/scala/org/bitcoins/wallet/db/WalletDbManagement.scala

@@ -42,9 +42,9 @@ trait WalletDbManagement extends DbManagement {
   override lazy val allTables: List[TableQuery[Table[_]]] = {
     List(accountTable,
          addressTable,
-         utxoTable,
          txTable,
          incomingTxTable,
+         utxoTable,
          outgoingTxTable)
   }
 

wallet/src/main/scala/org/bitcoins/wallet/models/AccountDAO.scala

@@ -13,8 +13,9 @@ case class AccountDAO()(
     override val appConfig: WalletAppConfig)
     extends CRUD[AccountDb, (HDCoin, Int)]
     with SlickUtil[AccountDb, (HDCoin, Int)] {
-  import org.bitcoins.db.DbCommonsColumnMappers._
   import profile.api._
+  private val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
+  import mappers._
 
   override val table: TableQuery[AccountTable] = TableQuery[AccountTable]
 
@@ -59,8 +60,6 @@ case class AccountDAO()(
   class AccountTable(tag: Tag)
       extends Table[AccountDb](tag, "wallet_accounts") {
 
-    import org.bitcoins.db.DbCommonsColumnMappers._
-
     def xpub: Rep[ExtPublicKey] = column[ExtPublicKey]("xpub")
 
     def purpose: Rep[HDPurpose] = column[HDPurpose]("hd_purpose")

wallet/src/main/scala/org/bitcoins/wallet/models/AddressDAO.scala

@@ -33,7 +33,8 @@ case class AddressDAO()(
 ) extends CRUD[AddressDb, BitcoinAddress]
     with SlickUtil[AddressDb, BitcoinAddress] {
   import profile.api._
-  import org.bitcoins.db.DbCommonsColumnMappers._
+  private val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
+  import mappers._
 
   override val table: profile.api.TableQuery[AddressTable] =
     TableQuery[AddressTable]
@@ -171,7 +172,6 @@ case class AddressDAO()(
     * todo: https://github.com/bitcoin-s/bitcoin-s-core/pull/391#discussion_r274188334
     */
   class AddressTable(tag: Tag) extends Table[AddressDb](tag, "addresses") {
-    import org.bitcoins.db.DbCommonsColumnMappers._
 
     def purpose: Rep[HDPurpose] = column("hd_purpose")
 

wallet/src/main/scala/org/bitcoins/wallet/models/IncomingTransactionDAO.scala

@@ -24,7 +24,8 @@ case class IncomingTransactionDAO()(
   class IncomingTransactionTable(tag: Tag)
       extends TxTable[IncomingTransactionDb](tag, "wallet_incoming_txs") {
 
-    import org.bitcoins.db.DbCommonsColumnMappers._
+    private val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
+    import mappers._
 
     def txIdBE: Rep[DoubleSha256DigestBE] = column("txIdBE", O.Unique)
 
@@ -44,7 +45,7 @@ case class IncomingTransactionDAO()(
       (txIdBE, incomingAmount) <> (fromTuple, toTuple)
 
     def primaryKey: PrimaryKey =
-      primaryKey("pk_tx", sourceColumns = txIdBE)
+      primaryKey("pk_in_tx", sourceColumns = txIdBE)
 
     def fk_underlying_tx: slick.lifted.ForeignKeyQuery[_, TransactionDb] = {
       foreignKey("fk_underlying_tx",

wallet/src/main/scala/org/bitcoins/wallet/models/OutgoingTransactionDAO.scala

@@ -26,7 +26,8 @@ case class OutgoingTransactionDAO()(
   class OutgoingTransactionTable(tag: Tag)
       extends TxTable[OutgoingTransactionDb](tag, "wallet_outgoing_txs") {
 
-    import org.bitcoins.db.DbCommonsColumnMappers._
+    private val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
+    import mappers._
 
     def txIdBE: Rep[DoubleSha256DigestBE] = column("txIdBE", O.Unique)
 
@@ -73,7 +74,7 @@ case class OutgoingTransactionDAO()(
       (txIdBE, inputAmount, sentAmount, actualFee, expectedFee, feeRate) <> (fromTuple, toTuple)
 
     def primaryKey: PrimaryKey =
-      primaryKey("pk_tx", sourceColumns = txIdBE)
+      primaryKey("pk_out_tx", sourceColumns = txIdBE)
 
     def fk_underlying_tx: slick.lifted.ForeignKeyQuery[_, TransactionDb] = {
       foreignKey("fk_underlying_tx",

wallet/src/main/scala/org/bitcoins/wallet/models/SpendingInfoDAO.scala

@@ -24,8 +24,9 @@ case class SpendingInfoDAO()(
     implicit val ec: ExecutionContext,
     override val appConfig: WalletAppConfig)
     extends CRUDAutoInc[SpendingInfoDb] {
-  import org.bitcoins.db.DbCommonsColumnMappers._
   import profile.api._
+  private val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
+  import mappers._
 
   /** The table inside our database we are inserting into */
   override val table: profile.api.TableQuery[SpendingInfoTable] =
@@ -176,7 +177,6 @@ case class SpendingInfoDAO()(
     */
   case class SpendingInfoTable(tag: Tag)
       extends TableAutoInc[SpendingInfoDb](tag, "txo_spending_info") {
-    import org.bitcoins.db.DbCommonsColumnMappers._
 
     def outPoint: Rep[TransactionOutPoint] =
       column("tx_outpoint")

wallet/src/main/scala/org/bitcoins/wallet/models/TransactionDAO.scala

@@ -29,7 +29,8 @@ trait TxDAO[DbEntryType <: TxDB]
   import profile.api._
   implicit val ec: ExecutionContext
 
-  import org.bitcoins.db.DbCommonsColumnMappers._
+  private val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
+  import mappers._
 
   type DbTable = TxTable[DbEntryType]
   override val table: TableQuery[_ <: DbTable]
@@ -76,14 +77,14 @@ case class TransactionDAO()(
     extends TxDAO[TransactionDb] {
 
   import profile.api._
+  private val mappers = new org.bitcoins.db.DbCommonsColumnMappers(profile)
+  import mappers._
 
   override val table = TableQuery[TransactionTable]
 
   class TransactionTable(tag: Tag)
       extends TxTable[TransactionDb](tag, "tx_table") {
 
-    import org.bitcoins.db.DbCommonsColumnMappers._
-
     def txIdBE: Rep[DoubleSha256DigestBE] = column("txIdBE", O.Unique)
 
     def transaction: Rep[Transaction] = column("transaction")