From 172744f47c5783a4a022848ca62fb292425a9b63 Mon Sep 17 00:00:00 2001 From: Chris Stewart Date: Mon, 24 Aug 2020 12:55:04 -0500 Subject: [PATCH] Db conf on pr 1877 (#1879) * Extend StartStopAsync with BitcoinSAppConfig, create 'CachedAppConfig' test trait, clean up P2PClientTest * Start cleaning up after ourselves in the chainTest test suite * Call .stop() for appConfig's spun up in ChainAppConfigTest * Database configuration defaults * increase number of Postgres connections * add more logging * close connections pools in tests * update afterAll() * Fix conflict Co-authored-by: rorp --- .../scala/org/bitcoins/db/DBConfigTest.scala | 106 ++++++++++++++++++ .../node/networking/P2PClientTest.scala | 1 + project/CommonSettings.scala | 4 +- .../testkit/BitcoinSTestAppConfig.scala | 2 +- .../org/bitcoins/testkit/EmbeddedPg.scala | 17 ++- .../bitcoins/testkit/db/TestDAOFixture.scala | 3 +- .../testkit/fixtures/WalletDAOFixture.scala | 12 +- .../bitcoins/testkit/node/NodeUnitTest.scala | 5 +- .../testkit/wallet/BitcoinSWalletTest.scala | 19 ++-- 9 files changed, 145 insertions(+), 24 deletions(-) create mode 100644 db-commons-test/src/test/scala/org/bitcoins/db/DBConfigTest.scala diff --git a/db-commons-test/src/test/scala/org/bitcoins/db/DBConfigTest.scala b/db-commons-test/src/test/scala/org/bitcoins/db/DBConfigTest.scala new file mode 100644 index 0000000000..04dce80881 --- /dev/null +++ b/db-commons-test/src/test/scala/org/bitcoins/db/DBConfigTest.scala @@ -0,0 +1,106 @@ +package org.bitcoins.db + +import java.io.{File, IOException} +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.{ + FileVisitResult, + Files, + Path, + SimpleFileVisitor, + StandardOpenOption +} + +import org.bitcoins.chain.config.ChainAppConfig +import org.bitcoins.node.config.NodeAppConfig +import org.bitcoins.wallet.config.WalletAppConfig +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class DBConfigTest extends AnyFlatSpec with Matchers { + + import scala.concurrent.ExecutionContext.Implicits.global + + it should "use sqlite as default database and set its connection pool size to 1" in { + withTempDir { dataDir => + val bytes = Files.readAllBytes( + new File("db-commons/src/main/resources/db.conf").toPath) + Files.write(dataDir.resolve("bitcoin-s.conf"), + bytes, + StandardOpenOption.CREATE_NEW, + StandardOpenOption.WRITE) + + val chainConfig = ChainAppConfig(dataDir) + val slickChainConfig = chainConfig.slickDbConfig + assert(slickChainConfig.profileName == "slick.jdbc.SQLiteProfile") + assert(slickChainConfig.config.hasPath("db.numThreads")) + assert(slickChainConfig.config.getInt("db.numThreads") == 1) + + val nodeConfig = NodeAppConfig(dataDir) + val slickNodeConfig = nodeConfig.slickDbConfig + assert(slickNodeConfig.profileName == "slick.jdbc.SQLiteProfile") + assert(slickNodeConfig.config.hasPath("db.numThreads")) + assert(slickNodeConfig.config.getInt("db.numThreads") == 1) + + val walletConfig = WalletAppConfig(dataDir) + val slickWalletConfig = walletConfig.slickDbConfig + assert(slickWalletConfig.profileName == "slick.jdbc.SQLiteProfile") + assert(slickWalletConfig.config.hasPath("db.numThreads")) + assert(slickWalletConfig.config.getInt("db.numThreads") == 1) + } + } + + it should "use sqlite as default database and disable connection pool for tests" in { + withTempDir { dataDir => + val chainConfig = ChainAppConfig(dataDir) + val slickChainConfig = chainConfig.slickDbConfig + assert(slickChainConfig.profileName == "slick.jdbc.SQLiteProfile") + assert(slickChainConfig.config.hasPath("db.numThreads")) + assert(slickChainConfig.config.getInt("db.numThreads") == 3) + assert( + slickChainConfig.config.getString("db.connectionPool") == "disabled") + + val nodeConfig = NodeAppConfig(dataDir) + val slickNodeConfig = nodeConfig.slickDbConfig + assert(slickNodeConfig.profileName == "slick.jdbc.SQLiteProfile") + assert(slickNodeConfig.config.hasPath("db.numThreads")) + assert(slickNodeConfig.config.getInt("db.numThreads") == 3) + assert( + slickNodeConfig.config.getString("db.connectionPool") == "disabled") + + val walletConfig = WalletAppConfig(dataDir) + val slickWalletConfig = walletConfig.slickDbConfig + assert(slickWalletConfig.profileName == "slick.jdbc.SQLiteProfile") + assert(slickWalletConfig.config.hasPath("db.numThreads")) + assert(slickWalletConfig.config.getInt("db.numThreads") == 3) + assert( + slickWalletConfig.config.getString("db.connectionPool") == "disabled") + } + } + + def withTempDir[T](f: Path => T): T = { + val dir = Files.createTempDirectory(getClass.getName) + try { + f(dir) + } finally { + Files.walkFileTree( + dir, + new SimpleFileVisitor[Path] { + override def visitFile( + file: Path, + attrs: BasicFileAttributes): FileVisitResult = { + Files.delete(file); + FileVisitResult.CONTINUE + } + + override def postVisitDirectory( + dir: Path, + exc: IOException): FileVisitResult = { + Files.delete(dir); + FileVisitResult.CONTINUE + } + } + ) + () + } + } +} diff --git a/node-test/src/test/scala/org/bitcoins/node/networking/P2PClientTest.scala b/node-test/src/test/scala/org/bitcoins/node/networking/P2PClientTest.scala index 6164ed2d31..059b141ce7 100644 --- a/node-test/src/test/scala/org/bitcoins/node/networking/P2PClientTest.scala +++ b/node-test/src/test/scala/org/bitcoins/node/networking/P2PClientTest.scala @@ -19,6 +19,7 @@ import org.bitcoins.testkit.node.{ CachedBitcoinSAppConfig, NodeTestUtil } +import org.bitcoins.testkit.node.{CachedAppConfig, NodeTestUtil} import org.bitcoins.testkit.rpc.BitcoindRpcTestUtil import org.bitcoins.testkit.util.BitcoindRpcTest import org.scalatest._ diff --git a/project/CommonSettings.scala b/project/CommonSettings.scala index f506b799cf..2bb476c6ff 100644 --- a/project/CommonSettings.scala +++ b/project/CommonSettings.scala @@ -114,9 +114,9 @@ object CommonSettings { lazy val testWithDbSettings: Seq[Setting[_]] = Seq( // To make in-memory DBs work properly - Test / fork := true, + Test / fork := false, // To avoid deadlock issues with SQLite - Test / parallelExecution := false + Test / parallelExecution := true ) ++ testSettings lazy val prodSettings: Seq[Setting[_]] = settings diff --git a/testkit/src/main/scala/org/bitcoins/testkit/BitcoinSTestAppConfig.scala b/testkit/src/main/scala/org/bitcoins/testkit/BitcoinSTestAppConfig.scala index 408f7fbd1d..4a0ab7c62e 100644 --- a/testkit/src/main/scala/org/bitcoins/testkit/BitcoinSTestAppConfig.scala +++ b/testkit/src/main/scala/org/bitcoins/testkit/BitcoinSTestAppConfig.scala @@ -113,7 +113,7 @@ object BitcoinSTestAppConfig { | driver = "org.postgresql.Driver" | username = "postgres" | password = "" - | connectionPool = disabled + | numThreads = 10 | keepAliveConnection = true | }""".stripMargin } diff --git a/testkit/src/main/scala/org/bitcoins/testkit/EmbeddedPg.scala b/testkit/src/main/scala/org/bitcoins/testkit/EmbeddedPg.scala index efc30119af..53983678a2 100644 --- a/testkit/src/main/scala/org/bitcoins/testkit/EmbeddedPg.scala +++ b/testkit/src/main/scala/org/bitcoins/testkit/EmbeddedPg.scala @@ -44,14 +44,19 @@ trait EmbeddedPg extends BeforeAndAfterAll { this: Suite => def executePgSql(sql: String): Unit = pg.foreach { pg => - val conn = pg.getPostgresDatabase.getConnection try { - val st = conn.createStatement() + val conn = pg.getPostgresDatabase.getConnection try { - st.execute(sql) - } finally st.close() - - } finally conn.close() + val st = conn.createStatement() + try { + st.execute(sql) + } finally st.close() + } finally conn.close() + } catch { + case ex: Throwable => + println(sql) + ex.printStackTrace() + } } } diff --git a/testkit/src/main/scala/org/bitcoins/testkit/db/TestDAOFixture.scala b/testkit/src/main/scala/org/bitcoins/testkit/db/TestDAOFixture.scala index c7efa031d5..12bf9aea3e 100644 --- a/testkit/src/main/scala/org/bitcoins/testkit/db/TestDAOFixture.scala +++ b/testkit/src/main/scala/org/bitcoins/testkit/db/TestDAOFixture.scala @@ -29,9 +29,8 @@ sealed trait TestDAOFixture } override def afterAll(): Unit = { - super.afterAll() testConfig.stop() - () + super.afterAll() } def withFixture(test: OneArgAsyncTest): FutureOutcome = { diff --git a/testkit/src/main/scala/org/bitcoins/testkit/fixtures/WalletDAOFixture.scala b/testkit/src/main/scala/org/bitcoins/testkit/fixtures/WalletDAOFixture.scala index a7663b5a03..f8e5dfedfc 100644 --- a/testkit/src/main/scala/org/bitcoins/testkit/fixtures/WalletDAOFixture.scala +++ b/testkit/src/main/scala/org/bitcoins/testkit/fixtures/WalletDAOFixture.scala @@ -44,17 +44,21 @@ trait WalletDAOFixture extends BitcoinSWalletTest { override def afterAll(): Unit = { super.afterAll() - walletConfig.stop() - () } def withFixture(test: OneArgAsyncTest): FutureOutcome = makeFixture(build = () => Future(walletConfig.migrate()).map(_ => daos), destroy = () => dropAll())(test) - def dropAll(): Future[Unit] = - for { + def dropAll(): Future[Unit] = { + val res = for { _ <- walletConfig.dropTable("flyway_schema_history") _ <- walletConfig.dropAll() } yield () + res.failed.foreach { ex => + ex.printStackTrace() + } + res + } + } diff --git a/testkit/src/main/scala/org/bitcoins/testkit/node/NodeUnitTest.scala b/testkit/src/main/scala/org/bitcoins/testkit/node/NodeUnitTest.scala index ab7b84f70d..c8528b0f58 100644 --- a/testkit/src/main/scala/org/bitcoins/testkit/node/NodeUnitTest.scala +++ b/testkit/src/main/scala/org/bitcoins/testkit/node/NodeUnitTest.scala @@ -48,7 +48,7 @@ import org.bitcoins.wallet.WalletCallbacks import org.scalatest.FutureOutcome import scala.concurrent.duration._ -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.{Await, ExecutionContext, Future} trait NodeUnitTest extends BitcoinSFixture with EmbeddedPg { @@ -58,6 +58,9 @@ trait NodeUnitTest extends BitcoinSFixture with EmbeddedPg { } override def afterAll(): Unit = { + Await.result(config.chainConf.stop(), 1.minute) + Await.result(config.nodeConf.stop(), 1.minute) + Await.result(config.walletConf.stop(), 1.minute) super[EmbeddedPg].afterAll() } diff --git a/testkit/src/main/scala/org/bitcoins/testkit/wallet/BitcoinSWalletTest.scala b/testkit/src/main/scala/org/bitcoins/testkit/wallet/BitcoinSWalletTest.scala index 3418b872c4..62651d67df 100644 --- a/testkit/src/main/scala/org/bitcoins/testkit/wallet/BitcoinSWalletTest.scala +++ b/testkit/src/main/scala/org/bitcoins/testkit/wallet/BitcoinSWalletTest.scala @@ -2,10 +2,10 @@ package org.bitcoins.testkit.wallet import akka.actor.ActorSystem import com.typesafe.config.{Config, ConfigFactory} -import org.bitcoins.core.api.chain.ChainQueryApi.FilterResponse -import org.bitcoins.core.api.node.NodeApi import org.bitcoins.core.api.chain.ChainQueryApi +import org.bitcoins.core.api.chain.ChainQueryApi.FilterResponse import org.bitcoins.core.api.feeprovider.FeeRateApi +import org.bitcoins.core.api.node.NodeApi import org.bitcoins.core.currency._ import org.bitcoins.core.gcs.BlockFilter import org.bitcoins.core.protocol.BlockStamp @@ -29,12 +29,8 @@ import org.bitcoins.wallet.config.WalletAppConfig import org.bitcoins.wallet.{Wallet, WalletCallbacks, WalletLogger} import org.scalatest._ -import scala.concurrent.{ - ExecutionContext, - ExecutionContextExecutor, - Future, - Promise -} +import scala.concurrent.duration._ +import scala.concurrent._ trait BitcoinSWalletTest extends BitcoinSFixture @@ -55,6 +51,13 @@ trait BitcoinSWalletTest super[EmbeddedPg].beforeAll() } + override def afterAll(): Unit = { + Await.result(config.chainConf.stop(), 1.minute) + Await.result(config.nodeConf.stop(), 1.minute) + Await.result(config.walletConf.stop(), 1.minute) + super[EmbeddedPg].afterAll() + } + def nodeApi: NodeApi = MockNodeApi val legacyWalletConf: Config =