Add configurable logging to data directory (#640)

* Add logging to data directory

In this commit we add the ability for the node, chain
and wallet projects (+ the server) to log to the users
data directory instead of whatever directory the
binaries was launched from. This is inherently a bit
more complicated than our previous setup, because
we need to read the user directory before we can create
loggers. As a result of this, some files/methods were
moved around, so the relevant app config could be
found in scope.

We also  introduce several logging categories that can be
tuned individually through user configuration. These logggers
are exposed both as traits that give a field `logger`, or
as methods that return the required logger.

* Add datadir configuration to AppConfig

In this commit we add support for AppConfig to pick up
the data directory configuration file. We also add
a section to the contributing guide on how to tune
logging levels.

* Pass data directories explicitly for configuration

app/server/src/main/resources/application.conf

@@ -1,4 +1,3 @@
 akka {
-    loggers = ["akka.event.slf4j.Slf4jLogger"]
+    loglevel = "INFO"
-    loglevel = "DEBUG"
 }

app/server/src/main/resources/logback.xml

@@ -0,0 +1,13 @@
+<configuration>  
+    <!-- see how long statements took to execute by setting to DEBUG -->
+    <logger name="slick.jdbc.JdbcBackend.benchmark" level="INFO"/>
+
+    <!-- see what statements are executed by setting to DEBUG -->
+    <logger name="slick.jdbc.JdbcBackend.statement" level="INFO"/>
+
+    <!-- see what slick is compiling to in sql -->
+    <logger name="slick.compiler" level="INFO"/>
+
+    <!-- see what's returned by Slick -->
+    <logger name="slick.jdbc.StatementInvoker.result" level="INFO"/>
+</configuration>

app/server/src/main/scala/org/bitcoins/server/BitcoinSAppConfig.scala

@@ -6,6 +6,8 @@ import org.bitcoins.node.config.NodeAppConfig
 import org.bitcoins.chain.config.ChainAppConfig
 import scala.concurrent.ExecutionContext
 import scala.concurrent.Future
+import java.nio.file.Path
+import org.bitcoins.db.AppConfig
 
 /**
   * A unified config class for all submodules of Bitcoin-S
@@ -13,11 +15,16 @@ import scala.concurrent.Future
   * in this case class' companion object an instance
   * of this class can be passed in anywhere a wallet,
   * chain or node config is required.
+  *
+  * @param directory The data directory of this app configuration
+  * @param confs A sequence of optional configuration overrides
   */
-case class BitcoinSAppConfig(private val confs: Config*) {
+case class BitcoinSAppConfig(
-  val walletConf = WalletAppConfig(confs: _*)
+    private val directory: Path,
-  val nodeConf = NodeAppConfig(confs: _*)
+    private val confs: Config*) {
-  val chainConf = ChainAppConfig(confs: _*)
+  val walletConf = WalletAppConfig(directory, confs: _*)
+  val nodeConf = NodeAppConfig(directory, confs: _*)
+  val chainConf = ChainAppConfig(directory, confs: _*)
 
   /** Initializes the wallet, node and chain projects */
   def initialize()(implicit ec: ExecutionContext): Future[Unit] = {
@@ -44,6 +51,13 @@ case class BitcoinSAppConfig(private val confs: Config*) {
   * to be passed in wherever a specializes one is required
   */
 object BitcoinSAppConfig {
+
+  /** Constructs an app configuration from the default Bitcoin-S
+    * data directory and given list of configuration overrides.
+    */
+  def fromDefaultDatadir(confs: Config*): BitcoinSAppConfig =
+    BitcoinSAppConfig(AppConfig.DEFAULT_BITCOIN_S_DATADIR, confs: _*)
+
   import scala.language.implicitConversions
 
   /** Converts the given implicit config to a wallet config */

app/server/src/main/scala/org/bitcoins/server/ChainRoutes.scala

@@ -4,14 +4,12 @@ import akka.actor.ActorSystem
 import akka.http.scaladsl.server._
 import akka.http.scaladsl.server.Directives._
 import akka.stream.ActorMaterializer
-import org.bitcoins.core.util.BitcoinSLogger
 import org.bitcoins.chain.api.ChainApi
 
 import org.bitcoins.picklers._
 
 case class ChainRoutes(chain: ChainApi)(implicit system: ActorSystem)
-    extends BitcoinSLogger
+    extends ServerRoute {
-    with ServerRoute {
   implicit val materializer = ActorMaterializer()
   import system.dispatcher
 

app/server/src/main/scala/org/bitcoins/server/Main.scala

@@ -2,7 +2,6 @@ package org.bitcoins.server
 
 import org.bitcoins.rpc.config.BitcoindInstance
 import org.bitcoins.node.models.Peer
-import org.bitcoins.core.util.BitcoinSLogger
 import org.bitcoins.rpc.client.common.BitcoindRpcClient
 import akka.actor.ActorSystem
 import scala.concurrent.Await
@@ -17,7 +16,6 @@ import org.bitcoins.wallet.api.InitializeWalletSuccess
 import org.bitcoins.wallet.api.InitializeWalletError
 import org.bitcoins.node.SpvNode
 import org.bitcoins.chain.blockchain.ChainHandler
-import org.bitcoins.chain.models.BlockHeaderDAO
 import org.bitcoins.chain.config.ChainAppConfig
 import org.bitcoins.wallet.api.UnlockedWalletApi
 import org.bitcoins.wallet.api.UnlockWalletSuccess
@@ -25,17 +23,19 @@ import org.bitcoins.wallet.api.UnlockWalletError
 import org.bitcoins.node.networking.peer.DataMessageHandler
 import org.bitcoins.node.SpvNodeCallbacks
 import org.bitcoins.wallet.WalletStorage
+import org.bitcoins.db.AppLoggers
+import org.bitcoins.chain.models.BlockHeaderDAO
 
-object Main
+object Main extends App {
-    extends App
-    // TODO we want to log to user data directory
-    // how do we do this?
-    with BitcoinSLogger {
   implicit val conf = {
     // val custom = ConfigFactory.parseString("bitcoin-s.network = testnet3")
-    BitcoinSAppConfig()
+    BitcoinSAppConfig.fromDefaultDatadir()
   }
 
+  private val logger = AppLoggers.getHttpLogger(
+    conf.walletConf // doesn't matter which one we pass in
+  )
+
   implicit val walletConf: WalletAppConfig = conf.walletConf
   implicit val nodeConf: NodeAppConfig = conf.nodeConf
   implicit val chainConf: ChainAppConfig = conf.chainConf
@@ -113,7 +113,7 @@ object Main
         SpvNodeCallbacks(onTxReceived = Seq(onTX))
       }
       val blockheaderDAO = BlockHeaderDAO()
-      val chain = ChainHandler(blockheaderDAO, conf)
+      val chain = ChainHandler(blockheaderDAO)
       SpvNode(peer, chain, bloom, callbacks).start()
     }
     _ = logger.info(s"Starting SPV node sync")
@@ -123,7 +123,9 @@ object Main
       val walletRoutes = WalletRoutes(wallet, node)
       val nodeRoutes = NodeRoutes(node)
       val chainRoutes = ChainRoutes(node.chainApi)
-      val server = Server(Seq(walletRoutes, nodeRoutes, chainRoutes))
+      val server =
+        Server(nodeConf, // could use either of configurations
+               Seq(walletRoutes, nodeRoutes, chainRoutes))
       server.start()
     }
   } yield start

app/server/src/main/scala/org/bitcoins/server/NodeRoutes.scala

@@ -4,12 +4,10 @@ import akka.actor.ActorSystem
 import akka.http.scaladsl.server._
 import akka.http.scaladsl.server.Directives._
 import akka.stream.ActorMaterializer
-import org.bitcoins.core.util.BitcoinSLogger
 import org.bitcoins.node.SpvNode
 
 case class NodeRoutes(node: SpvNode)(implicit system: ActorSystem)
-    extends BitcoinSLogger
+    extends ServerRoute {
-    with ServerRoute {
   implicit val materializer = ActorMaterializer()
 
   def handleCommand: PartialFunction[ServerCommand, StandardRoute] = {

app/server/src/main/scala/org/bitcoins/server/Server.scala

@@ -4,7 +4,6 @@ import upickle.{default => up}
 import akka.actor.ActorSystem
 import akka.http.scaladsl._
 import akka.stream.ActorMaterializer
-import org.bitcoins.core.util.BitcoinSLogger
 import akka.http.scaladsl.model._
 import akka.http.scaladsl.server._
 import akka.http.scaladsl.server.Directives._
@@ -12,9 +11,14 @@ import akka.http.scaladsl.server.Directives._
 import de.heikoseeberger.akkahttpupickle.UpickleSupport._
 import akka.http.scaladsl.server.directives.DebuggingDirectives
 import akka.event.Logging
+import org.bitcoins.db.HttpLogger
+import org.bitcoins.db.AppConfig
+
+case class Server(conf: AppConfig, handlers: Seq[ServerRoute])(
+    implicit system: ActorSystem)
+    extends HttpLogger {
+  implicit private val config: AppConfig = conf
 
-case class Server(handlers: Seq[ServerRoute])(implicit system: ActorSystem)
-    extends BitcoinSLogger {
   implicit val materializer = ActorMaterializer()
   import system.dispatcher
 
@@ -83,7 +87,7 @@ case class Server(handlers: Seq[ServerRoute])(implicit system: ActorSystem)
   }
 }
 
-object Server extends BitcoinSLogger {
+object Server {
 
   // TODO id parameter
   case class Response(

app/server/src/main/scala/org/bitcoins/server/WalletRoutes.scala

@@ -7,7 +7,6 @@ import akka.http.scaladsl.server._
 import akka.http.scaladsl.server.Directives._
 import akka.stream.ActorMaterializer
 
-import org.bitcoins.core.util.BitcoinSLogger
 import org.bitcoins.core.currency._
 import org.bitcoins.wallet.api.UnlockedWalletApi
 import org.bitcoins.core.wallet.fee.SatoshisPerByte
@@ -19,8 +18,7 @@ import scala.util.Success
 
 case class WalletRoutes(wallet: UnlockedWalletApi, node: SpvNode)(
     implicit system: ActorSystem)
-    extends BitcoinSLogger
+    extends ServerRoute {
-    with ServerRoute {
   import system.dispatcher
   implicit val materializer = ActorMaterializer()
 

chain-test/src/test/scala/org/bitcoins/chain/ChainAppConfigTest.scala

@@ -7,9 +7,12 @@ import com.typesafe.config.ConfigFactory
 import org.bitcoins.core.config.RegTest
 import org.bitcoins.core.config.MainNet
 import org.bitcoins.chain.config.ChainAppConfig
+import java.nio.file.Files
+import ch.qos.logback.classic.Level
 
 class ChainAppConfigTest extends BitcoinSUnitTest {
-  val config = ChainAppConfig()
+  val tempDir = Files.createTempDirectory("bitcoin-s")
+  val config = ChainAppConfig(directory = tempDir)
 
   it must "be overridable" in {
     assert(config.network == RegTest)
@@ -30,4 +33,29 @@ class ChainAppConfigTest extends BitcoinSUnitTest {
     assert(overriden.network == MainNet)
 
   }
+
+  it must "have user data directory configuration take precedence" in {
+
+    val tempDir = Files.createTempDirectory("bitcoin-s")
+    val tempFile = Files.createFile(tempDir.resolve("bitcoin-s.conf"))
+    val confStr = """
+    | bitcoin-s {
+    |   network = testnet3
+    |   
+    |   logging {
+    |     level = off
+    |
+    |     p2p = warn
+    |   }
+    | }
+    """.stripMargin
+    val _ = Files.write(tempFile, confStr.getBytes())
+
+    val appConfig = ChainAppConfig(directory = tempDir)
+
+    assert(appConfig.datadir == tempDir.resolve("testnet3"))
+    assert(appConfig.network == TestNet3)
+    assert(appConfig.logLevel == Level.OFF)
+    assert(appConfig.p2pLogLevel == Level.WARN)
+  }
 }

chain/src/main/scala/org/bitcoins/chain/api/ChainApi.scala

@@ -12,7 +12,7 @@ import org.bitcoins.chain.config.ChainAppConfig
   */
 trait ChainApi {
 
-  def chainConfig: ChainAppConfig
+  implicit private[chain] val chainConfig: ChainAppConfig
 
   /**
     * Adds a block header to our chain project

chain/src/main/scala/org/bitcoins/chain/blockchain/Blockchain.scala

@@ -3,10 +3,11 @@ package org.bitcoins.chain.blockchain
 import org.bitcoins.chain.models.{BlockHeaderDAO, BlockHeaderDb}
 import org.bitcoins.chain.validation.{TipUpdateResult, TipValidation}
 import org.bitcoins.core.protocol.blockchain.BlockHeader
-import org.bitcoins.core.util.BitcoinSLogger
 
 import scala.collection.{IndexedSeqLike, mutable}
 import scala.concurrent.{ExecutionContext, Future}
+import org.bitcoins.chain.config.ChainAppConfig
+import org.bitcoins.db.ChainVerificationLogger
 
 /**
   * In memory implementation of a blockchain
@@ -21,8 +22,7 @@ import scala.concurrent.{ExecutionContext, Future}
   *
   */
 case class Blockchain(headers: Vector[BlockHeaderDb])
-    extends IndexedSeqLike[BlockHeaderDb, Vector[BlockHeaderDb]]
+    extends IndexedSeqLike[BlockHeaderDb, Vector[BlockHeaderDb]] {
-    with BitcoinSLogger {
   val tip: BlockHeaderDb = headers.head
 
   /** @inheritdoc */
@@ -41,7 +41,7 @@ case class Blockchain(headers: Vector[BlockHeaderDb])
 
 }
 
-object Blockchain extends BitcoinSLogger {
+object Blockchain extends ChainVerificationLogger {
 
   def fromHeaders(headers: Vector[BlockHeaderDb]): Blockchain = {
     Blockchain(headers)
@@ -61,7 +61,8 @@ object Blockchain extends BitcoinSLogger {
     *         or [[org.bitcoins.chain.blockchain.BlockchainUpdate.Failed Failed]] to connect to a tip
     */
   def connectTip(header: BlockHeader, blockHeaderDAO: BlockHeaderDAO)(
-      implicit ec: ExecutionContext): Future[BlockchainUpdate] = {
+      implicit ec: ExecutionContext,
+      conf: ChainAppConfig): Future[BlockchainUpdate] = {
 
     //get all competing chains we have
     val blockchainsF: Future[Vector[Blockchain]] =

chain/src/main/scala/org/bitcoins/chain/blockchain/ChainHandler.scala

@@ -5,20 +5,19 @@ import org.bitcoins.chain.config.ChainAppConfig
 import org.bitcoins.chain.models.{BlockHeaderDAO, BlockHeaderDb}
 import org.bitcoins.core.crypto.DoubleSha256DigestBE
 import org.bitcoins.core.protocol.blockchain.BlockHeader
-import org.bitcoins.core.util.BitcoinSLogger
 
 import scala.concurrent.{ExecutionContext, Future}
+import org.bitcoins.db.ChainVerificationLogger
 
 /**
   * Chain Handler is meant to be the reference implementation
   * of [[org.bitcoins.chain.api.ChainApi ChainApi]], this is the entry point in to the
   * chain project.
   */
-case class ChainHandler(
+case class ChainHandler(blockHeaderDAO: BlockHeaderDAO)(
-    blockHeaderDAO: BlockHeaderDAO,
+    implicit private[chain] val chainConfig: ChainAppConfig
-    chainConfig: ChainAppConfig)
+) extends ChainApi
-    extends ChainApi
+    with ChainVerificationLogger {
-    with BitcoinSLogger {
 
   override def getBlockCount(implicit ec: ExecutionContext): Future[Long] = {
     logger.debug(s"Querying for block count")
@@ -43,7 +42,8 @@ case class ChainHandler(
   override def processHeader(header: BlockHeader)(
       implicit ec: ExecutionContext): Future[ChainHandler] = {
 
-    val blockchainUpdateF = Blockchain.connectTip(header, blockHeaderDAO)
+    val blockchainUpdateF =
+      Blockchain.connectTip(header, blockHeaderDAO)
 
     val newHandlerF = blockchainUpdateF.flatMap {
       case BlockchainUpdate.Successful(_, updatedHeader) =>
@@ -53,7 +53,7 @@ case class ChainHandler(
         createdF.map { header =>
           logger.debug(
             s"Connected new header to blockchain, height=${header.height} hash=${header.hashBE}")
-          ChainHandler(blockHeaderDAO, chainConfig)
+          ChainHandler(blockHeaderDAO)
         }
       case BlockchainUpdate.Failed(_, _, reason) =>
         val errMsg =

chain/src/main/scala/org/bitcoins/chain/blockchain/sync/ChainSync.scala

@@ -5,11 +5,12 @@ import org.bitcoins.chain.blockchain.ChainHandler
 import org.bitcoins.chain.models.BlockHeaderDb
 import org.bitcoins.core.crypto.DoubleSha256DigestBE
 import org.bitcoins.core.protocol.blockchain.BlockHeader
-import org.bitcoins.core.util.BitcoinSLogger
 
 import scala.concurrent.{ExecutionContext, Future}
+import org.bitcoins.chain.config.ChainAppConfig
+import org.bitcoins.db.ChainVerificationLogger
 
-trait ChainSync extends BitcoinSLogger {
+trait ChainSync extends ChainVerificationLogger {
 
   /** This method checks if our chain handler has the tip of the blockchain as an external source
     * If we do not have the same chain, we sync our chain handler until we are at the same best block hash
@@ -20,9 +21,12 @@ trait ChainSync extends BitcoinSLogger {
     * @param ec
     * @return
     */
-  def sync(chainHandler: ChainHandler,
+  def sync(
+      chainHandler: ChainHandler,
       getBlockHeaderFunc: DoubleSha256DigestBE => Future[BlockHeader],
-           getBestBlockHashFunc: () => Future[DoubleSha256DigestBE])(implicit ec: ExecutionContext): Future[ChainApi] = {
+      getBestBlockHashFunc: () => Future[DoubleSha256DigestBE])(
+      implicit ec: ExecutionContext,
+      conf: ChainAppConfig): Future[ChainApi] = {
     val currentTipsF: Future[Vector[BlockHeaderDb]] = {
       chainHandler.blockHeaderDAO.chainTips
     }
@@ -50,7 +54,6 @@ trait ChainSync extends BitcoinSLogger {
 
   }
 
-
   /**
     * Keeps walking backwards on the chain until we match one
     * of the tips we have in our chain
@@ -61,17 +64,22 @@ trait ChainSync extends BitcoinSLogger {
     * @param ec
     * @return
     */
-  private def syncTips(chainApi: ChainApi,
+  private def syncTips(
+      chainApi: ChainApi,
       tips: Vector[BlockHeaderDb],
       bestBlockHash: DoubleSha256DigestBE,
-                       getBlockHeaderFunc: DoubleSha256DigestBE => Future[BlockHeader])(implicit ec: ExecutionContext): Future[ChainApi] = {
+      getBlockHeaderFunc: DoubleSha256DigestBE => Future[BlockHeader])(
+      implicit ec: ExecutionContext,
+      conf: ChainAppConfig): Future[ChainApi] = {
     require(tips.nonEmpty, s"Cannot sync without the genesis block")
 
     //we need to walk backwards on the chain until we get to one of our tips
 
     val tipsBH = tips.map(_.blockHeader)
 
-    def loop(lastHeaderF: Future[BlockHeader], accum: List[BlockHeader]): Future[List[BlockHeader]] = {
+    def loop(
+        lastHeaderF: Future[BlockHeader],
+        accum: List[BlockHeader]): Future[List[BlockHeader]] = {
       lastHeaderF.flatMap { lastHeader =>
         if (tipsBH.contains(lastHeader)) {
           //means we have synced back to a block that we know
@@ -81,9 +89,10 @@ trait ChainSync extends BitcoinSLogger {
           logger.debug(s"Last header=${lastHeader.hashBE.hex}")
           //we don't know this block, so we need to keep walking backwards
           //to find a block a we know
-          val newLastHeaderF = getBlockHeaderFunc(lastHeader.previousBlockHashBE)
+          val newLastHeaderF =
+            getBlockHeaderFunc(lastHeader.previousBlockHashBE)
 
-          loop(newLastHeaderF,lastHeader +: accum)
+          loop(newLastHeaderF, lastHeader +: accum)
         }
       }
     }
@@ -91,7 +100,9 @@ trait ChainSync extends BitcoinSLogger {
     val bestHeaderF = getBlockHeaderFunc(bestBlockHash)
 
     bestHeaderF.map { bestHeader =>
-      logger.info(s"Best tip from third party=${bestHeader.hashBE.hex} currentTips=${tips.map(_.hashBE.hex)}")
+      logger.info(
+        s"Best tip from third party=${bestHeader.hashBE.hex} currentTips=${tips
+          .map(_.hashBE.hex)}")
     }
 
     //one sanity check to make sure we aren't _ahead_ of our data source
@@ -110,7 +121,8 @@ trait ChainSync extends BitcoinSLogger {
 
         //now we are going to add them to our chain and return the chain api
         headersToSyncF.flatMap { headers =>
-          logger.info(s"Attempting to sync ${headers.length} blockheader to our chainstate")
+          logger.info(
+            s"Attempting to sync ${headers.length} blockheader to our chainstate")
           chainApi.processHeaders(headers.toVector)
         }
       }
@@ -120,5 +132,4 @@ trait ChainSync extends BitcoinSLogger {
   }
 }
 
-
 object ChainSync extends ChainSync

chain/src/main/scala/org/bitcoins/chain/config/ChainAppConfig.scala

@@ -11,13 +11,24 @@ import scala.concurrent.Future
 import scala.concurrent.Promise
 import scala.util.Success
 import scala.util.Failure
+import java.nio.file.Path
 
-case class ChainAppConfig(private val confs: Config*) extends AppConfig {
+/** Configuration for the Bitcoin-S chain verification module
-  override protected val configOverrides: List[Config] = confs.toList
+  * @param directory The data directory of the module
-  override protected val moduleName: String = "chain"
+  * @param confs Optional sequence of configuration overrides
-  override protected type ConfigType = ChainAppConfig
+  */
-  override protected def newConfigOfType(configs: Seq[Config]): ChainAppConfig =
+case class ChainAppConfig(
-    ChainAppConfig(configs: _*)
+    private val directory: Path,
+    private val confs: Config*)
+    extends AppConfig {
+  override protected[bitcoins] def configOverrides: List[Config] = confs.toList
+  override protected[bitcoins] val moduleName: String = "chain"
+  override protected[bitcoins] type ConfigType = ChainAppConfig
+  override protected[bitcoins] def newConfigOfType(
+      configs: Seq[Config]): ChainAppConfig =
+    ChainAppConfig(directory, configs: _*)
+
+  protected[bitcoins] def baseDatadir: Path = directory
 
   /**
     * Checks whether or not the chain project is initialized by
@@ -70,3 +81,12 @@ case class ChainAppConfig(private val confs: Config*) extends AppConfig {
     }
   }
 }
+
+object ChainAppConfig {
+
+  /** Constructs a chain verification configuration from the default Bitcoin-S
+    * data directory and given list of configuration overrides.
+    */
+  def fromDefaultDatadir(confs: Config*): ChainAppConfig =
+    ChainAppConfig(AppConfig.DEFAULT_BITCOIN_S_DATADIR, confs: _*)
+}

chain/src/main/scala/org/bitcoins/chain/models/BlockHeaderDAO.scala

@@ -16,8 +16,8 @@ import scala.concurrent.{ExecutionContext, Future}
   * our chain project
   */
 case class BlockHeaderDAO()(
-    implicit override val ec: ExecutionContext,
+    implicit ec: ExecutionContext,
-    override val appConfig: ChainAppConfig)
+    appConfig: ChainAppConfig)
     extends CRUD[BlockHeaderDb, DoubleSha256DigestBE] {
 
   import org.bitcoins.db.DbCommonsColumnMappers._

chain/src/main/scala/org/bitcoins/chain/pow/Pow.scala

@@ -2,8 +2,9 @@ package org.bitcoins.chain.pow
 
 import org.bitcoins.chain.models.{BlockHeaderDAO, BlockHeaderDb}
 import org.bitcoins.core.number.UInt32
+import org.bitcoins.chain.config.ChainAppConfig
 import org.bitcoins.core.protocol.blockchain.{BlockHeader, ChainParams}
-import org.bitcoins.core.util.{BitcoinSLogger, NumberUtil}
+import org.bitcoins.core.util.NumberUtil
 
 import scala.concurrent.{ExecutionContext, Future}
 
@@ -11,7 +12,7 @@ import scala.concurrent.{ExecutionContext, Future}
   * Implements functions found inside of bitcoin core's
   * @see [[https://github.com/bitcoin/bitcoin/blob/35477e9e4e3f0f207ac6fa5764886b15bf9af8d0/src/pow.cpp pow.cpp]]
   */
-sealed abstract class Pow extends BitcoinSLogger {
+sealed abstract class Pow {
 
   /**
     * Gets the next proof of work requirement for a block
@@ -24,8 +25,9 @@ sealed abstract class Pow extends BitcoinSLogger {
       tip: BlockHeaderDb,
       newPotentialTip: BlockHeader,
       blockHeaderDAO: BlockHeaderDAO)(
-      implicit ec: ExecutionContext): Future[UInt32] = {
+      implicit ec: ExecutionContext,
-    val chainParams = blockHeaderDAO.appConfig.chain
+      config: ChainAppConfig): Future[UInt32] = {
+    val chainParams = config.chain
     val currentHeight = tip.height
 
     val powLimit = NumberUtil.targetCompression(bigInteger =

chain/src/main/scala/org/bitcoins/chain/validation/TipValidation.scala

@@ -8,9 +8,11 @@ import org.bitcoins.chain.models.{
 import org.bitcoins.chain.pow.Pow
 import org.bitcoins.core.number.UInt32
 import org.bitcoins.core.protocol.blockchain.BlockHeader
-import org.bitcoins.core.util.{BitcoinSLogger, NumberUtil}
+import org.bitcoins.core.util.NumberUtil
 
 import scala.concurrent.{ExecutionContext, Future}
+import org.bitcoins.chain.config.ChainAppConfig
+import org.bitcoins.db.ChainVerificationLogger
 
 /**
   * Responsible for checking if we can connect two
@@ -18,7 +20,7 @@ import scala.concurrent.{ExecutionContext, Future}
   * things like proof of work difficulty, if it
   * references the previous block header correctly etc.
   */
-sealed abstract class TipValidation extends BitcoinSLogger {
+sealed abstract class TipValidation extends ChainVerificationLogger {
 
   /** Checks if the given header can be connected to the current tip
     * This is the method where a [[org.bitcoins.core.protocol.blockchain.BlockHeader BlockHeader]] is transformed into a
@@ -30,7 +32,8 @@ sealed abstract class TipValidation extends BitcoinSLogger {
       newPotentialTip: BlockHeader,
       currentTip: BlockHeaderDb,
       blockHeaderDAO: BlockHeaderDAO)(
-      implicit ec: ExecutionContext): Future[TipUpdateResult] = {
+      implicit ec: ExecutionContext,
+      conf: ChainAppConfig): Future[TipUpdateResult] = {
     val header = newPotentialTip
     logger.trace(
       s"Checking header=${header.hashBE.hex} to try to connect to currentTip=${currentTip.hashBE.hex} with height=${currentTip.height}")
@@ -67,7 +70,9 @@ sealed abstract class TipValidation extends BitcoinSLogger {
   /** Logs the result of [[org.bitcoins.chain.validation.TipValidation.checkNewTip() checkNewTip]] */
   private def logTipResult(
       connectTipResultF: Future[TipUpdateResult],
-      currentTip: BlockHeaderDb)(implicit ec: ExecutionContext): Unit = {
+      currentTip: BlockHeaderDb)(
+      implicit ec: ExecutionContext,
+      conf: ChainAppConfig): Unit = {
     connectTipResultF.map {
       case TipUpdateResult.Success(tipDb) =>
         logger.trace(
@@ -102,7 +107,8 @@ sealed abstract class TipValidation extends BitcoinSLogger {
       newPotentialTip: BlockHeader,
       currentTip: BlockHeaderDb,
       blockHeaderDAO: BlockHeaderDAO)(
-      implicit ec: ExecutionContext): Future[UInt32] = {
+      implicit ec: ExecutionContext,
+      config: ChainAppConfig): Future[UInt32] = {
     Pow.getNetworkWorkRequired(tip = currentTip,
                                newPotentialTip = newPotentialTip,
                                blockHeaderDAO = blockHeaderDAO)

core-test/src/test/scala/org/bitcoins/core/p2p/MerkleBlockMessageTest.scala

@@ -3,8 +3,8 @@ package org.bitcoins.core.p2p
 import org.bitcoins.testkit.core.gen.p2p.DataMessageGenerator
 import org.bitcoins.testkit.util.BitcoinSUnitTest
 import scodec.bits._
-import org.bitcoins.node.util.BitcoinSpvNodeUtil
 import org.bitcoins.core.crypto.DoubleSha256Digest
+import org.bitcoins.node.networking.P2PClient
 
 class MerkleBlockMessageTest extends BitcoinSUnitTest {
   it must "have serialization symmetry" in {
@@ -42,15 +42,4 @@ class MerkleBlockMessageTest extends BitcoinSUnitTest {
     assert(fourth == expectedFourth)
   }
 
-  // we had a bug where we didn't consume the right number of bytes
-  // when parsing a merkle block message, thereby screwing up
-  // the parsing of the remainder
-  it must "parse a byte vector with three messages in it" in {
-    val bytes =
-      hex"fabfb5da6d65726b6c65626c6f636b0097000000b4b6e45d00000020387191f7d488b849b4080fdf105c71269fc841a2f0f2944fc5dc785c830c716e37f36373098aae06a668cc74e388caf50ecdcb5504ce936490b4b72940e08859548c305dffff7f20010000000200000002ecd1c722709bfc241f8b94fc64034dcba2c95409dc4cd1d7b864e1128a04e5b044133327b04ff8ac576e7748a4dae4111f0c765dacbfe0c5a9fddbeb8f60d5af0105fabfb5da747800000000000000000000cc0100004413332702000000065b7f0f3eec398047e921037815aa41709b6243a1897f1423194b7558399ae0300000000017160014008dc9d88d1797305f3fbd30d2b36d6bde984a09feffffffe9145055d671fd705a09f028033da614b619205b9926fe5ebe45e15ae8b3231e0100000017160014d74cfac04bb0e6838c35f1f4a0a60d13655be2fbfeffffff797f8ff9c10fa618b6254343a648be995410e82c03fd8accb0de2271a3fb1abd00000000171600143ee832c09db48eca28a64a358ed7a01dbe52d31bfeffffffc794dba971b9479dfcbc662a3aacd641553bdb2418b15c0221c5dfd4471a7a70000000001716001452c13ba0314f7718c234ed6adfea6422ce03a545feffffffb7c3bf1762b15f3b0e0eaa5beb46fe96a9e2829a7413fd900b9b7e0d192ab64800000000171600143ee832c09db48eca28a64a358ed7a01dbe52d31bfeffffffb6ced6cb8dfc2f7f5b37561938ead3bc5ca4036e2b45d9738cc086a10eed4e010100000017160014aebb17e245fe8c98a75f0b6717fcadca30e491e2feffffff02002a7515000000001976a9148374ff8beb55ea2945039881ca26071b5749fafe88ac485620000000000017a91405d36a2b0bdedf3fc58bed6f9e4026f8934a2716876b050000fabfb5da686561646572730000000000010000001406e05800"
-    val (messages, leftover) = BitcoinSpvNodeUtil.parseIndividualMessages(bytes)
-    assert(messages.length == 3)
-    assert(leftover.isEmpty)
-
-  }
 }

db-commons/src/main/resources/reference.conf

@@ -2,6 +2,41 @@ bitcoin-s {
     datadir = ${HOME}/.bitcoin-s
     network = regtest # regtest, testnet3, mainnet
 
+    logging {
+        level = info # trace, debug, info, warn, error, off
+
+        # You can also tune specific module loggers.
+        # They each take the same levels as above.
+        # If they are commented out (as they are
+        # by default), `logging.level` gets used
+        # instead. 
+        # The available loggers are: 
+
+        # incoming and outgoing P2P messages
+        # p2p = info
+
+        # verification of block headers, merkle trees
+        # chain-verification = info
+
+        # generation of addresses, signing of TXs
+        # key-handling = info
+
+        # wallet operations not related to key management
+        # wallet = info
+
+        # HTTP RPC server
+        # http = info
+
+        # Database interactions
+        # database = info
+
+        # whether or not to write to the log file
+        disable-file = false
+
+        # whether or not to log to stdout 
+        disable-console = false
+    }
+
     # settings for wallet module
     wallet {
         defaultAccountType = legacy # legacy, segwit, nested-segwit

db-commons/src/main/scala/org/bitcoins/db/AppConfig.scala

@@ -26,6 +26,7 @@ import scala.util.Properties
 import scala.util.matching.Regex
 import scala.concurrent.ExecutionContext
 import scala.concurrent.Future
+import ch.qos.logback.classic.Level
 
 /**
   * Everything needed to configure functionality
@@ -51,15 +52,16 @@ abstract class AppConfig extends BitcoinSLogger {
     * the type of themselves, ensuring `withOverrides` return
     * the correct type
     */
-  protected type ConfigType <: AppConfig
+  protected[bitcoins] type ConfigType <: AppConfig
 
   /** Constructor to make a new instance of this config type */
-  protected def newConfigOfType(configOverrides: Seq[Config]): ConfigType
+  protected[bitcoins] def newConfigOfType(
+      configOverrides: Seq[Config]): ConfigType
 
   /** List of user-provided configs that should
     * override defaults
     */
-  protected val configOverrides: List[Config] = List.empty
+  protected[bitcoins] def configOverrides: List[Config] = List.empty
 
   /**
     * This method returns a new `AppConfig`, where every
@@ -121,7 +123,7 @@ abstract class AppConfig extends BitcoinSLogger {
   /**
     * Name of the module. `chain`, `wallet`, `node` etc.
     */
-  protected def moduleName: String
+  protected[bitcoins] def moduleName: String
 
   /**
     * The configuration details for connecting/using the database for our projects
@@ -206,6 +208,27 @@ abstract class AppConfig extends BitcoinSLogger {
     * rest of the fields in this class from
     */
   private[bitcoins] lazy val config: Config = {
+
+    val datadirConfig = {
+      val file = baseDatadir.resolve("bitcoin-s.conf")
+      val config = if (Files.isReadable(file)) {
+        ConfigFactory.parseFile(file.toFile())
+      } else {
+        ConfigFactory.empty()
+      }
+
+      val withDatadir =
+        ConfigFactory.parseString(s"bitcoin-s.datadir = $baseDatadir")
+      withDatadir.withFallback(config)
+    }
+
+    logger.trace(s"Data directory config:")
+    if (datadirConfig.hasPath("bitcoin-s")) {
+      logger.trace(datadirConfig.getConfig("bitcoin-s").asReadableJson)
+    } else {
+      logger.trace(ConfigFactory.empty().asReadableJson)
+    }
+
     // `load` tries to resolve substitions,
     // `parseResources` does not
     val dbConfig = ConfigFactory
@@ -226,9 +249,12 @@ abstract class AppConfig extends BitcoinSLogger {
     logger.trace(
       s"Classpath config: ${classPathConfig.getConfig("bitcoin-s").asReadableJson}")
 
-    // loads reference.conf as well as application.conf,
+    // we want the data directory configuration
-    // if the user has made one
+    // to take preference over any bundled (classpath)
-    val unresolvedConfig = classPathConfig
+    // configurations
+    // loads reference.conf (provided by Bitcoin-S)
+    val unresolvedConfig = datadirConfig
+      .withFallback(classPathConfig)
       .withFallback(dbConfig)
 
     logger.trace(s"Unresolved bitcoin-s config:")
@@ -256,32 +282,92 @@ abstract class AppConfig extends BitcoinSLogger {
         unresolvedConfig
       }
 
-    val config = withOverrides
+    val finalConfig = withOverrides
       .resolve()
       .getConfig("bitcoin-s")
 
     logger.debug(s"Resolved bitcoin-s config:")
-    logger.debug(config.asReadableJson)
+    logger.debug(finalConfig.asReadableJson)
 
-    config
+    finalConfig
 
   }
 
-  /** The data directory used by bitcoin-s apps */
+  /** The base data directory. This is where we look for a configuration file */
-  lazy val datadir: Path = {
+  protected[bitcoins] def baseDatadir: Path
-    val basedir = Paths.get(config.getString("datadir"))
+
+  /** The network specific data directory. */
+  val datadir: Path = {
     val lastDirname = network match {
       case MainNet  => "mainnet"
       case TestNet3 => "testnet3"
       case RegTest  => "regtest"
     }
-    basedir.resolve(lastDirname)
+    baseDatadir.resolve(lastDirname)
   }
 
+  private def stringToLogLevel(str: String): Level =
+    str.toLowerCase() match {
+      case "trace"       => Level.TRACE
+      case "debug"       => Level.DEBUG
+      case "info"        => Level.INFO
+      case "warn"        => Level.WARN
+      case "error"       => Level.ERROR
+      case "off"         => Level.OFF
+      case other: String => sys.error(s"Unknown logging level: $other")
+    }
+
+  /** The default logging level */
+  lazy val logLevel: Level = {
+    val levelString = config.getString("logging.level")
+    stringToLogLevel(levelString)
+  }
+
+  /** Whether or not we should log to file */
+  lazy val disableFileLogging = config.getBoolean("logging.disable-file")
+
+  /** Whether or not we should log to stdout */
+  lazy val disableConsoleLogging = config.getBoolean("logging.disable-console")
+
+  private def levelOrDefault(key: String): Level =
+    config
+      .getStringOrNone(key)
+      .map(stringToLogLevel)
+      .getOrElse(logLevel)
+
+  /** The logging level for our P2P logger */
+  lazy val p2pLogLevel: Level = levelOrDefault("logging.p2p")
+
+  /** The logging level for our chain verification logger */
+  lazy val verificationLogLevel: Level =
+    levelOrDefault("logging.chain-verification")
+
+  /** The logging level for our key handling logger */
+  lazy val keyHandlingLogLevel: Level =
+    levelOrDefault("logging.key-handling")
+
+  /** Logging level for wallet */
+  lazy val walletLogLeveL: Level =
+    levelOrDefault("logging.wallet")
+
+  /** Logging level for HTTP RPC server */
+  lazy val httpLogLevel: Level = levelOrDefault("logging.http")
+
+  /** Logging level for database interactions */
+  lazy val databaseLogLevel: Level = levelOrDefault("logging.database")
+
 }
 
 object AppConfig extends BitcoinSLogger {
 
+  /** The default data directory
+    *
+    * TODO: use different directories on Windows and Mac,
+    * should probably mimic what Bitcoin Core does
+    */
+  private[bitcoins] val DEFAULT_BITCOIN_S_DATADIR: Path =
+    Paths.get(Properties.userHome, ".bitcoin-s")
+
   /**
     * Matches the default data directory location
     * with a network appended,

db-commons/src/main/scala/org/bitcoins/db/CRUD.scala

@@ -1,6 +1,5 @@
 package org.bitcoins.db
 
-import org.bitcoins.core.util.BitcoinSLogger
 import slick.jdbc.SQLiteProfile.api._
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -15,16 +14,16 @@ import org.bitcoins.core.config.MainNet
   * You are responsible for the create function. You also need to specify
   * the table and the database you are connecting to.
   */
-abstract class CRUD[T, PrimaryKeyType] extends BitcoinSLogger {
+abstract class CRUD[T, PrimaryKeyType](
-
+    implicit private val config: AppConfig,
-  def appConfig: AppConfig
+    private val ec: ExecutionContext)
-  implicit val ec: ExecutionContext
+    extends DatabaseLogger {
 
   /** The table inside our database we are inserting into */
   val table: TableQuery[_ <: Table[T]]
 
   /** Binding to the actual database itself, this is what is used to run querys */
-  def database: SafeDatabase = SafeDatabase(appConfig)
+  def database: SafeDatabase = SafeDatabase(config)
 
   /**
     * create a record in the database
@@ -33,7 +32,7 @@ abstract class CRUD[T, PrimaryKeyType] extends BitcoinSLogger {
     * @return the inserted record
     */
   def create(t: T): Future[T] = {
-    logger.trace(s"Writing $t to DB with config: ${appConfig.config}")
+    logger.trace(s"Writing $t to DB with config: ${config.config}")
     createAll(Vector(t)).map(_.head)
   }
 
@@ -46,7 +45,7 @@ abstract class CRUD[T, PrimaryKeyType] extends BitcoinSLogger {
     * @return Option[T] - the record if found, else none
     */
   def read(id: PrimaryKeyType): Future[Option[T]] = {
-    logger.trace(s"Reading from DB with config: ${appConfig.config}")
+    logger.trace(s"Reading from DB with config: ${config.config}")
     val query = findByPrimaryKey(id)
     val rows: Future[Seq[T]] = database.run(query.result)
     rows.map(_.headOption)
@@ -130,7 +129,8 @@ abstract class CRUD[T, PrimaryKeyType] extends BitcoinSLogger {
 
 }
 
-case class SafeDatabase(config: AppConfig) extends BitcoinSLogger {
+case class SafeDatabase(config: AppConfig) extends DatabaseLogger {
+  implicit private val conf: AppConfig = config
 
   import config.database
 

db-commons/src/main/scala/org/bitcoins/db/CRUDAutoInc.scala

@@ -4,8 +4,12 @@ import slick.dbio.Effect.Write
 import slick.jdbc.SQLiteProfile.api._
 
 import scala.concurrent.Future
+import scala.concurrent.ExecutionContext
 
-abstract class CRUDAutoInc[T <: DbRowAutoInc[T]] extends CRUD[T, Long] {
+abstract class CRUDAutoInc[T <: DbRowAutoInc[T]](
+    implicit config: AppConfig,
+    ec: ExecutionContext)
+    extends CRUD[T, Long] {
 
   /** The table inside our database we are inserting into */
   override val table: TableQuery[_ <: TableAutoInc[T]]

db-commons/src/main/scala/org/bitcoins/db/DbManagement.scala

@@ -1,11 +1,10 @@
 package org.bitcoins.db
 
-import org.bitcoins.core.util.BitcoinSLogger
 import slick.jdbc.SQLiteProfile.api._
 
 import scala.concurrent.{ExecutionContext, Future}
 
-abstract class DbManagement extends BitcoinSLogger {
+abstract class DbManagement extends DatabaseLogger {
   def allTables: List[TableQuery[_ <: Table[_]]]
 
   /** Lists all tables in the given database */

db-commons/src/main/scala/org/bitcoins/db/package..scala

@@ -11,6 +11,24 @@ package object db {
       val options = ConfigRenderOptions.concise().setFormatted(true)
       config.root().render(options)
     }
+
+    /** Returns the string at key or the given default value */
+    def getStringOrElse(key: String, default: => String): String = {
+      if (config.hasPath(key)) {
+        config.getString(key)
+      } else {
+        default
+      }
+    }
+
+    /** Returns the string at the given key, if it exists */
+    def getStringOrNone(key: String): Option[String] = {
+      if (config.hasPath(key)) {
+        Some(config.getString(key))
+      } else {
+        None
+      }
+    }
   }
 
 }

docs/contributing.md

@@ -13,6 +13,42 @@ It's possible to communicate with other developers through a variety of communic
 - [Bitcoin-S Gitter](https://gitter.im/bitcoin-s-core/)
 - [#bitcoin-scala](https://webchat.freenode.net/?channels=bitcoin-scala) on IRC Freenode
 
+## Working on Bitcoin-S applications
+
+Bitcoin-S includes a couple of applications that can be run as standalone executables.
+This includes the node, wallet and (partial) blockchain verification modules, as well
+as the server that bundles these three together and the CLI used to communicate with
+the server. These applications are configured with HOCON files. The file
+[`reference.conf`](https://github.com/bitcoin-s/bitcoin-s/blob/master/db-commons/src/main/resources/reference.conf)
+is the basis configuration file, and every option read by Bitcoin-S should be present in
+this file. This means that you can copy sections from this file and edit them, to tune
+how the application runs on your machine.
+
+One example of things you can tune is logging levels. Lets say you wanted general logging
+to happen at the `WARN` level, but the P2P message handling to be logged at `DEBUG`. Your
+configuration file would then look like:
+
+```conf
+bitcoins-s {
+  logging {
+    level = warn
+
+    p2p = debug
+  }
+}
+```
+
+### Running the applications
+
+When running the applications configuration placed in `bitcoin-s.conf` in the current
+data directory gets picked up. For linux this is by default `$HOME/.bitcoin-s/`, so the
+file you should edit would be `$HOME/.bitcoin-s/bitcoin-s.conf`.
+
+### Running tests for the applications
+
+You can place configuration files in the data directory that tests are being run in,
+but you can also edit [`reference.conf`](https://github.com/bitcoin-s/bitcoin-s/blob/master/db-commons/src/main/resources/reference.conf).
+
 ## Developer productivity
 
 ### Bloop

node-test/src/test/scala/org/bitcoins/node/BroadcastTransactionTest.scala

@@ -61,7 +61,7 @@ class BroadcastTransactionTest extends BitcoinSWalletTest {
         val peer = Peer.fromBitcoind(rpc.instance)
         val chainHandler = {
           val bhDao = BlockHeaderDAO()
-          ChainHandler(bhDao, config)
+          ChainHandler(bhDao)
         }
 
         val spv =

node-test/src/test/scala/org/bitcoins/node/NodeAppConfigTest.scala

@@ -7,9 +7,12 @@ import com.typesafe.config.Config
 import com.typesafe.config.ConfigFactory
 import org.bitcoins.core.config.RegTest
 import org.bitcoins.core.config.MainNet
+import ch.qos.logback.classic.Level
+import java.nio.file.Files
 
 class NodeAppConfigTest extends BitcoinSUnitTest {
-  val config = NodeAppConfig()
+  val tempDir = Files.createTempDirectory("bitcoin-s")
+  val config = NodeAppConfig(directory = tempDir)
 
   it must "be overridable" in {
     assert(config.network == RegTest)
@@ -30,4 +33,29 @@ class NodeAppConfigTest extends BitcoinSUnitTest {
     assert(overriden.network == MainNet)
 
   }
+
+  it must "have user data directory configuration take precedence" in {
+
+    val tempDir = Files.createTempDirectory("bitcoin-s")
+    val tempFile = Files.createFile(tempDir.resolve("bitcoin-s.conf"))
+    val confStr = """
+    | bitcoin-s {
+    |   network = testnet3
+    |   
+    |   logging {
+    |     level = off
+    |
+    |     p2p = warn
+    |   }
+    | }
+    """.stripMargin
+    val _ = Files.write(tempFile, confStr.getBytes())
+
+    val appConfig = NodeAppConfig(directory = tempDir)
+
+    assert(appConfig.datadir == tempDir.resolve("testnet3"))
+    assert(appConfig.network == TestNet3)
+    assert(appConfig.logLevel == Level.OFF)
+    assert(appConfig.p2pLogLevel == Level.WARN)
+  }
 }

node-test/src/test/scala/org/bitcoins/node/NodeWithWalletTest.scala

@@ -107,7 +107,7 @@ class NodeWithWalletTest extends BitcoinSWalletTest {
           val peer = Peer.fromBitcoind(rpc.instance)
           val chainHandler = {
             val bhDao = BlockHeaderDAO()
-            ChainHandler(bhDao, config)
+            ChainHandler(bhDao)
           }
 
           val spv =

node-test/src/test/scala/org/bitcoins/node/networking/P2PClientTest.scala

@@ -12,14 +12,24 @@ import org.bitcoins.testkit.node.NodeTestUtil
 import org.bitcoins.testkit.rpc.BitcoindRpcTestUtil
 import org.bitcoins.testkit.util.BitcoindRpcTest
 import org.scalatest._
+import scodec.bits._
 
 import scala.concurrent.Future
 import scala.concurrent.duration.DurationInt
+import org.bitcoins.core.p2p.HeadersMessage
+import org.bitcoins.core.protocol.CompactSizeUInt
+import org.bitcoins.core.number.UInt64
+import org.bitcoins.core.protocol.blockchain.BlockHeader
+import org.bitcoins.core.number.Int32
+import org.bitcoins.core.crypto.DoubleSha256Digest
+import org.bitcoins.core.number.UInt32
+import org.bitcoins.core.p2p.NetworkMessage
+import org.bitcoins.core.p2p.VersionMessage
+import org.bitcoins.core.config.TestNet3
+import org.bitcoins.chain.blockchain.ChainHandler
+import org.bitcoins.chain.models.BlockHeaderDAO
 
-/**
+class P2PClientTest
-  * Created by chris on 6/7/16.
-  */
-class ClientTest
     extends BitcoindRpcTest
     with MustMatchers
     with BeforeAndAfter
@@ -45,8 +55,76 @@ class ClientTest
   lazy val bitcoindPeer2F = bitcoindRpcF.map { bitcoind =>
     NodeTestUtil.getBitcoindPeer(bitcoind)
   }
+  behavior of "parseIndividualMessages"
 
-  behavior of "Client"
+  it must "block header message that is not aligned with a tcp frame" in {
+
+    val headersMsg = HeadersMessage(
+      CompactSizeUInt(UInt64(2), 1),
+      Vector(
+        BlockHeader(
+          Int32(315017594),
+          DoubleSha256Digest(
+            "177e777f078d2deeaa3ad4b82e78a00ad2f4738c5217f7a36d9cf3bd11e41817"),
+          DoubleSha256Digest(
+            "1dcaebebd620823bb344bd18a18276de508910d66b4e3cbb3426a14eced66224"),
+          UInt32(2845833462L),
+          UInt32(2626024374L),
+          UInt32(2637850613L)
+        ),
+        BlockHeader(
+          Int32(1694049746),
+          DoubleSha256Digest(
+            "07b6d61809476830bc7ef862a983a7222997df3f639e0d2aa5902a5a48018430"),
+          DoubleSha256Digest(
+            "68c65f803b70b72563e86ac3e8e20ad11fbfa2eac3f9fddf4bc624d03a14f084"),
+          UInt32(202993555),
+          UInt32(4046619225L),
+          UInt32(1231236881)
+        )
+      )
+    )
+    val networkMsg = NetworkMessage(np, headersMsg)
+    //split the network msg at a random index to simulate a tcp frame not being aligned
+    val randomIndex = scala.util.Random.nextInt().abs % networkMsg.bytes.size
+    val (firstHalf, secondHalf) = networkMsg.bytes.splitAt(randomIndex)
+    val (firstHalfParseHeaders, remainingBytes) =
+      P2PClient.parseIndividualMessages(firstHalf)
+    firstHalfParseHeaders must be(empty)
+
+    val (secondHalfParsedHeaders, _) =
+      P2PClient.parseIndividualMessages(remainingBytes ++ secondHalf)
+    val parsedNetworkMsg = secondHalfParsedHeaders.head
+    val parsedHeadersMsg = parsedNetworkMsg.payload.asInstanceOf[HeadersMessage]
+    parsedNetworkMsg.header must be(networkMsg.header)
+    parsedHeadersMsg.headers.head must be(headersMsg.headers.head)
+    parsedHeadersMsg.headers(1) must be(parsedHeadersMsg.headers(1))
+
+  }
+
+  it must "return the entire byte array if a message is not aligned to a byte frame" in {
+    val versionMessage =
+      VersionMessage(TestNet3.dnsSeeds(0), np)
+    val networkMsg = NetworkMessage(np, versionMessage)
+    //remove last byte so the message is not aligned
+    val bytes = networkMsg.bytes.slice(0, networkMsg.bytes.size - 1)
+    val (_, unAlignedBytes) = P2PClient.parseIndividualMessages(bytes)
+
+    unAlignedBytes must be(bytes)
+  }
+
+  // we had a bug where we didn't consume the right number of bytes
+  // when parsing a merkle block message, thereby screwing up
+  // the parsing of the remainder
+  it must "parse a byte vector with three messages in it" in {
+    val bytes =
+      hex"fabfb5da6d65726b6c65626c6f636b0097000000b4b6e45d00000020387191f7d488b849b4080fdf105c71269fc841a2f0f2944fc5dc785c830c716e37f36373098aae06a668cc74e388caf50ecdcb5504ce936490b4b72940e08859548c305dffff7f20010000000200000002ecd1c722709bfc241f8b94fc64034dcba2c95409dc4cd1d7b864e1128a04e5b044133327b04ff8ac576e7748a4dae4111f0c765dacbfe0c5a9fddbeb8f60d5af0105fabfb5da747800000000000000000000cc0100004413332702000000065b7f0f3eec398047e921037815aa41709b6243a1897f1423194b7558399ae0300000000017160014008dc9d88d1797305f3fbd30d2b36d6bde984a09feffffffe9145055d671fd705a09f028033da614b619205b9926fe5ebe45e15ae8b3231e0100000017160014d74cfac04bb0e6838c35f1f4a0a60d13655be2fbfeffffff797f8ff9c10fa618b6254343a648be995410e82c03fd8accb0de2271a3fb1abd00000000171600143ee832c09db48eca28a64a358ed7a01dbe52d31bfeffffffc794dba971b9479dfcbc662a3aacd641553bdb2418b15c0221c5dfd4471a7a70000000001716001452c13ba0314f7718c234ed6adfea6422ce03a545feffffffb7c3bf1762b15f3b0e0eaa5beb46fe96a9e2829a7413fd900b9b7e0d192ab64800000000171600143ee832c09db48eca28a64a358ed7a01dbe52d31bfeffffffb6ced6cb8dfc2f7f5b37561938ead3bc5ca4036e2b45d9738cc086a10eed4e010100000017160014aebb17e245fe8c98a75f0b6717fcadca30e491e2feffffff02002a7515000000001976a9148374ff8beb55ea2945039881ca26071b5749fafe88ac485620000000000017a91405d36a2b0bdedf3fc58bed6f9e4026f8934a2716876b050000fabfb5da686561646572730000000000010000001406e05800"
+    val (messages, leftover) = P2PClient.parseIndividualMessages(bytes)
+    assert(messages.length == 3)
+    assert(leftover.isEmpty)
+
+  }
+  behavior of "P2PClient"
 
   it must "establish a tcp connection with a bitcoin node" in {
     bitcoindPeerF.flatMap(remote => connectAndDisconnect(remote))
@@ -74,8 +152,12 @@ class ClientTest
   def connectAndDisconnect(peer: Peer): Future[Assertion] = {
     val probe = TestProbe()
     val remote = peer.socket
+    val chainHandler = {
+      val dao = BlockHeaderDAO()
+      ChainHandler(dao)
+    }
     val peerMessageReceiver =
-      PeerMessageReceiver(state = Preconnection)
+      PeerMessageReceiver(state = Preconnection, chainHandler)
     val client =
       TestActorRef(P2PClient.props(peer, peerMessageReceiver), probe.ref)
 

node/src/main/scala/org/bitcoins/node/SpvNode.scala

@@ -2,9 +2,6 @@ package org.bitcoins.node
 
 import akka.actor.ActorSystem
 import org.bitcoins.chain.api.ChainApi
-import org.bitcoins.chain.config.ChainAppConfig
-import org.bitcoins.core.util.BitcoinSLogger
-import org.bitcoins.node.config.NodeAppConfig
 import org.bitcoins.node.models.Peer
 import org.bitcoins.node.networking.P2PClient
 import org.bitcoins.node.networking.peer.{
@@ -23,23 +20,22 @@ import org.bitcoins.node.models.BroadcastAbleTransactionDAO
 import slick.jdbc.SQLiteProfile
 import scala.util.Failure
 import scala.util.Success
+import org.bitcoins.db.P2PLogger
+import org.bitcoins.node.config.NodeAppConfig
 
 case class SpvNode(
     peer: Peer,
     chainApi: ChainApi,
     bloomFilter: BloomFilter,
     callbacks: SpvNodeCallbacks = SpvNodeCallbacks.empty
-)(
+)(implicit system: ActorSystem, nodeAppConfig: NodeAppConfig)
-    implicit system: ActorSystem,
+    extends P2PLogger {
-    nodeAppConfig: NodeAppConfig,
-    chainAppConfig: ChainAppConfig)
-    extends BitcoinSLogger {
   import system.dispatcher
 
   private val txDAO = BroadcastAbleTransactionDAO(SQLiteProfile)
 
   private val peerMsgRecv =
-    PeerMessageReceiver.newReceiver(callbacks)
+    PeerMessageReceiver.newReceiver(chainApi, callbacks)
 
   private val client: P2PClient =
     P2PClient(context = system, peer = peer, peerMessageReceiver = peerMsgRecv)

node/src/main/scala/org/bitcoins/node/config/NodeAppConfig.scala

@@ -7,13 +7,24 @@ import scala.concurrent.Future
 import org.bitcoins.node.db.NodeDbManagement
 import scala.util.Failure
 import scala.util.Success
+import java.nio.file.Path
 
-case class NodeAppConfig(private val confs: Config*) extends AppConfig {
+/** Configuration for the Bitcoin-S node
-  override val configOverrides: List[Config] = confs.toList
+  * @param directory The data directory of the node
-  override protected def moduleName: String = "node"
+  * @param confs Optional sequence of configuration overrides
-  override protected type ConfigType = NodeAppConfig
+  */
-  override protected def newConfigOfType(configs: Seq[Config]): NodeAppConfig =
+case class NodeAppConfig(
-    NodeAppConfig(configs: _*)
+    private val directory: Path,
+    private val confs: Config*)
+    extends AppConfig {
+  override protected[bitcoins] def configOverrides: List[Config] = confs.toList
+  override protected[bitcoins] val moduleName: String = "node"
+  override protected[bitcoins] type ConfigType = NodeAppConfig
+  override protected[bitcoins] def newConfigOfType(
+      configs: Seq[Config]): NodeAppConfig =
+    NodeAppConfig(directory, configs: _*)
+
+  protected[bitcoins] def baseDatadir: Path = directory
 
   /**
     * Ensures correct tables and other required information is in
@@ -31,3 +42,13 @@ case class NodeAppConfig(private val confs: Config*) extends AppConfig {
     initF
   }
 }
+
+object NodeAppConfig {
+
+  /** Constructs a node configuration from the default Bitcoin-S
+    * data directory and given list of configuration overrides.
+    */
+  def fromDefaultDatadir(confs: Config*): NodeAppConfig =
+    NodeAppConfig(AppConfig.DEFAULT_BITCOIN_S_DATADIR, confs: _*)
+
+}

node/src/main/scala/org/bitcoins/node/networking/P2PClient.scala

@@ -5,7 +5,6 @@ import akka.io.{IO, Tcp}
 import akka.util.ByteString
 import org.bitcoins.core.config.NetworkParameters
 import org.bitcoins.core.p2p.NetworkMessage
-import org.bitcoins.core.util.BitcoinSLogger
 import org.bitcoins.core.p2p.NetworkPayload
 import org.bitcoins.node.models.Peer
 import org.bitcoins.node.networking.peer.PeerMessageReceiver
@@ -14,6 +13,9 @@ import org.bitcoins.node.util.BitcoinSpvNodeUtil
 import scodec.bits.ByteVector
 import org.bitcoins.node.config.NodeAppConfig
 import akka.util.CompactByteString
+import scala.annotation.tailrec
+import scala.util._
+import org.bitcoins.db.P2PLogger
 
 /**
   * This actor is responsible for creating a connection,
@@ -50,7 +52,7 @@ case class P2PClientActor(
     peerMsgHandlerReceiver: PeerMessageReceiver
 )(implicit config: NodeAppConfig)
     extends Actor
-    with BitcoinSLogger {
+    with P2PLogger {
 
   /**
     * The manager is an actor that handles the underlying low level I/O resources (selectors, channels)
@@ -179,7 +181,7 @@ case class P2PClientActor(
         val bytes: ByteVector = unalignedBytes ++ byteVec
         logger.trace(s"Bytes for message parsing: ${bytes.toHex}")
         val (messages, newUnalignedBytes) =
-          BitcoinSpvNodeUtil.parseIndividualMessages(bytes)
+          P2PClient.parseIndividualMessages(bytes)
 
         logger.debug({
           val length = messages.length
@@ -237,7 +239,7 @@ case class P2PClientActor(
 
 case class P2PClient(actor: ActorRef, peer: Peer)
 
-object P2PClient {
+object P2PClient extends P2PLogger {
 
   def props(peer: Peer, peerMsgHandlerReceiver: PeerMessageReceiver)(
       implicit config: NodeAppConfig
@@ -256,4 +258,48 @@ object P2PClient {
     P2PClient(actorRef, peer)
   }
 
+  /**
+    * Akka sends messages as one byte stream. There is not a 1 to 1 relationship between byte streams received and
+    * bitcoin protocol messages. This function parses our byte stream into individual network messages
+    *
+    * @param bytes the bytes that need to be parsed into individual messages
+    * @return the parsed [[NetworkMessage]]'s and the unaligned bytes that did not parse to a message
+    */
+  private[bitcoins] def parseIndividualMessages(bytes: ByteVector)(
+      implicit conf: NodeAppConfig): (List[NetworkMessage], ByteVector) = {
+    @tailrec
+    def loop(
+        remainingBytes: ByteVector,
+        accum: List[NetworkMessage]): (List[NetworkMessage], ByteVector) = {
+      if (remainingBytes.length <= 0) {
+        (accum.reverse, remainingBytes)
+      } else {
+        val messageTry = Try(NetworkMessage(remainingBytes))
+        messageTry match {
+          case Success(message) =>
+            if (message.header.payloadSize.toInt != message.payload.bytes.size) {
+              //this means our tcp frame was not aligned, therefore put the message back in the
+              //buffer and wait for the remaining bytes
+              (accum.reverse, remainingBytes)
+            } else {
+              val newRemainingBytes = remainingBytes.slice(
+                message.bytes.length,
+                remainingBytes.length)
+              loop(newRemainingBytes, message :: accum)
+            }
+          case Failure(exception) =>
+            logger.error(
+              "Failed to parse network message, could be because TCP frame isn't aligned",
+              exception)
+            //this case means that our TCP frame was not aligned with bitcoin protocol
+            //return the unaligned bytes so we can apply them to the next tcp frame of bytes we receive
+            //http://stackoverflow.com/a/37979529/967713
+            (accum.reverse, remainingBytes)
+        }
+      }
+    }
+    val (messages, remainingBytes) = loop(bytes, Nil)
+    (messages, remainingBytes)
+  }
+
 }

node/src/main/scala/org/bitcoins/node/networking/peer/DataMessageHandler.scala

@@ -1,10 +1,7 @@
 package org.bitcoins.node.networking.peer
 
 import org.bitcoins.chain.api.ChainApi
-import org.bitcoins.chain.blockchain.ChainHandler
+import org.bitcoins.core.util.FutureUtil
-import org.bitcoins.chain.config.ChainAppConfig
-import org.bitcoins.chain.models.BlockHeaderDAO
-import org.bitcoins.core.util.{BitcoinSLogger, FutureUtil}
 import org.bitcoins.core.p2p.{DataPayload, HeadersMessage, InventoryMessage}
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -21,21 +18,20 @@ import slick.jdbc.SQLiteProfile
 import org.bitcoins.node.config.NodeAppConfig
 import org.bitcoins.core.p2p.TypeIdentifier
 import org.bitcoins.core.p2p.MsgUnassigned
+import org.bitcoins.db.P2PLogger
 
 /** This actor is meant to handle a [[org.bitcoins.core.p2p.DataPayload DataPayload]]
   * that a peer to sent to us on the p2p network, for instance, if we a receive a
   * [[org.bitcoins.core.p2p.HeadersMessage HeadersMessage]] we should store those headers in our database
   */
-class DataMessageHandler(callbacks: SpvNodeCallbacks)(
+class DataMessageHandler(callbacks: SpvNodeCallbacks, chainHandler: ChainApi)(
     implicit ec: ExecutionContext,
-    chainConf: ChainAppConfig,
+    appConfig: NodeAppConfig)
-    nodeConf: NodeAppConfig)
+    extends P2PLogger {
-    extends BitcoinSLogger {
 
   private val callbackNum = callbacks.onBlockReceived.length + callbacks.onMerkleBlockReceived.length + callbacks.onTxReceived.length
   logger.debug(s"Given $callbackNum of callback(s)")
 
-  private val blockHeaderDAO: BlockHeaderDAO = BlockHeaderDAO()
   private val txDAO = BroadcastAbleTransactionDAO(SQLiteProfile)
 
   def handleDataPayload(
@@ -76,9 +72,7 @@ class DataMessageHandler(callbacks: SpvNodeCallbacks)(
         logger.trace(
           s"Received headers message with ${headersMsg.count.toInt} headers")
         val headers = headersMsg.headers
-        val chainApi: ChainApi =
+        val chainApiF = chainHandler.processHeaders(headers)
-          ChainHandler(blockHeaderDAO, chainConfig = chainConf)
-        val chainApiF = chainApi.processHeaders(headers)
 
         chainApiF.map { newApi =>
           val lastHeader = headers.last

node/src/main/scala/org/bitcoins/node/networking/peer/PeerMessageReceiver.scala

@@ -1,9 +1,7 @@
 package org.bitcoins.node.networking.peer
 
 import akka.actor.ActorRefFactory
-import org.bitcoins.chain.config.ChainAppConfig
 import org.bitcoins.core.p2p.NetworkMessage
-import org.bitcoins.core.util.BitcoinSLogger
 import org.bitcoins.node.config.NodeAppConfig
 import org.bitcoins.core.p2p._
 import org.bitcoins.node.models.Peer
@@ -17,6 +15,8 @@ import org.bitcoins.node.networking.peer.PeerMessageReceiverState.{
 
 import scala.util.{Failure, Success, Try}
 import org.bitcoins.node.SpvNodeCallbacks
+import org.bitcoins.db.P2PLogger
+import org.bitcoins.chain.api.ChainApi
 
 /**
   * Responsible for receiving messages from a peer on the
@@ -27,12 +27,10 @@ import org.bitcoins.node.SpvNodeCallbacks
   */
 class PeerMessageReceiver(
     state: PeerMessageReceiverState,
-    callbacks: SpvNodeCallbacks
+    callbacks: SpvNodeCallbacks,
-)(
+    chainHandler: ChainApi
-    implicit ref: ActorRefFactory,
+)(implicit ref: ActorRefFactory, nodeAppConfig: NodeAppConfig)
-    nodeAppConfig: NodeAppConfig,
+    extends P2PLogger {
-    chainAppConfig: ChainAppConfig)
-    extends BitcoinSLogger {
 
   import ref.dispatcher
 
@@ -138,7 +136,7 @@ class PeerMessageReceiver(
   private def handleDataPayload(
       payload: DataPayload,
       sender: PeerMessageSender): Unit = {
-    val dataMsgHandler = new DataMessageHandler(callbacks)
+    val dataMsgHandler = new DataMessageHandler(callbacks, chainHandler)
     //else it means we are receiving this data payload from a peer,
     //we need to handle it
     dataMsgHandler.handleDataPayload(payload, sender)
@@ -233,18 +231,20 @@ object PeerMessageReceiver {
 
   def apply(
       state: PeerMessageReceiverState,
+      chainHandler: ChainApi,
       callbacks: SpvNodeCallbacks = SpvNodeCallbacks.empty)(
       implicit ref: ActorRefFactory,
-      nodeAppConfig: NodeAppConfig,
+      nodeAppConfig: NodeAppConfig): PeerMessageReceiver = {
-      chainAppConfig: ChainAppConfig
+    new PeerMessageReceiver(state, callbacks, chainHandler)
-  ): PeerMessageReceiver = {
-    new PeerMessageReceiver(state, callbacks)
   }
 
-  def newReceiver(callbacks: SpvNodeCallbacks = SpvNodeCallbacks.empty)(
+  def newReceiver(
+      chainHandler: ChainApi,
+      callbacks: SpvNodeCallbacks = SpvNodeCallbacks.empty)(
       implicit nodeAppConfig: NodeAppConfig,
-      chainAppConfig: ChainAppConfig,
       ref: ActorRefFactory): PeerMessageReceiver = {
-    new PeerMessageReceiver(state = PeerMessageReceiverState.fresh(), callbacks)
+    new PeerMessageReceiver(state = PeerMessageReceiverState.fresh(),
+                            callbacks,
+                            chainHandler)
   }
 }

node/src/main/scala/org/bitcoins/node/networking/peer/PeerMessageReceiverState.scala

@@ -1,12 +1,11 @@
 package org.bitcoins.node.networking.peer
 
-import org.bitcoins.core.util.BitcoinSLogger
 import org.bitcoins.core.p2p.{VerAckMessage, VersionMessage}
 import org.bitcoins.node.networking.P2PClient
 
 import scala.concurrent.{Future, Promise}
 
-sealed abstract class PeerMessageReceiverState extends BitcoinSLogger {
+sealed abstract class PeerMessageReceiverState {
 
   /** This promise gets completed when we receive a
     * [[akka.io.Tcp.Connected]] message from [[org.bitcoins.node.networking.P2PClient P2PClient]]

node/src/main/scala/org/bitcoins/node/networking/peer/PeerMessageSender.scala

@@ -4,14 +4,14 @@ import akka.actor.ActorRef
 import akka.io.Tcp
 import org.bitcoins.core.crypto.DoubleSha256Digest
 import org.bitcoins.core.p2p.NetworkMessage
-import org.bitcoins.core.util.BitcoinSLogger
 import org.bitcoins.core.p2p._
 import org.bitcoins.node.networking.P2PClient
 import org.bitcoins.node.config.NodeAppConfig
 import org.bitcoins.core.protocol.transaction.Transaction
+import org.bitcoins.db.P2PLogger
 
 case class PeerMessageSender(client: P2PClient)(implicit conf: NodeAppConfig)
-    extends BitcoinSLogger {
+    extends P2PLogger {
   private val socket = client.peer.socket
 
   /** Initiates a connection with the given peer */

project/Deps.scala

@@ -30,6 +30,7 @@ object Deps {
     val uPickleV = "0.7.4"
     val akkaHttpUpickleV = "1.27.0"
     val uJsonV = uPickleV // Li Haoyi ecosystem does common versioning
+    val sourcecodeV = "0.1.7"
 
     // CLI deps
     val scoptV = "4.0.0-RC2"
@@ -44,7 +45,6 @@ object Deps {
     val akkaHttp = "com.typesafe.akka" %% "akka-http" % V.akkav withSources () withJavadoc ()
     val akkaStream = "com.typesafe.akka" %% "akka-stream" % V.akkaStreamv withSources () withJavadoc ()
     val akkaActor = "com.typesafe.akka" %% "akka-actor" % V.akkaStreamv withSources () withJavadoc ()
-    val akkaLog = "com.typesafe.akka" %% "akka-slf4j" % V.akkaStreamv
 
     val playJson = "com.typesafe.play" %% "play-json" % V.playv withSources () withJavadoc ()
     val typesafeConfig = "com.typesafe" % "config" % V.typesafeConfigV withSources () withJavadoc ()
@@ -65,6 +65,9 @@ object Deps {
     // serializing to and from JSON
     val uPickle = "com.lihaoyi" %% "upickle" % V.uPickleV
 
+    // get access to reflection data at compile-time
+    val sourcecode = "com.lihaoyi" %% "sourcecode" % V.sourcecodeV
+
     // make akka-http play nice with upickle
     val akkaHttpUpickle = "de.heikoseeberger" %% "akka-http-upickle" % V.akkaHttpUpickleV
 
@@ -93,12 +96,10 @@ object Deps {
   }
 
   val chain = List(
-    Compile.slf4j
+    Compile.logback
   )
 
-  val chainTest = List(
+  val chainTest = List()
-    Test.logback
-  )
 
   val core = List(
     Compile.bouncycastle,
@@ -151,6 +152,8 @@ object Deps {
 
   val dbCommons = List(
     Compile.slick,
+    Compile.sourcecode,
+    Compile.logback,
     Compile.sqlite,
     Compile.slickHikari
   )
@@ -169,7 +172,6 @@ object Deps {
     Compile.akkaHttpUpickle,
     Compile.uPickle,
     Compile.logback,
-    Compile.akkaLog,
     Compile.akkaHttp
   )
 
@@ -198,7 +200,6 @@ object Deps {
 
   val nodeTest = List(
     Test.akkaTestkit,
-    Test.logback,
     Test.scalaTest
   )
 
@@ -215,11 +216,11 @@ object Deps {
   )
 
   val wallet = List(
-    Compile.uJson
+    Compile.uJson,
+    Compile.logback
   )
 
   val walletTest = List(
-    Test.logback,
     Test.akkaTestkit
   )
 

testkit/src/main/scala/org/bitcoins/testkit/BitcoinSTestAppConfig.scala

@@ -13,15 +13,7 @@ object BitcoinSTestAppConfig {
     */
   def getTestConfig(config: Config*): BitcoinSAppConfig = {
     val tmpDir = Files.createTempDirectory("bitcoin-s-")
-    val confStr = s"""
+    BitcoinSAppConfig(tmpDir, config: _*)
-    | bitcoin-s {
-    |   datadir = $tmpDir
-    | }
-    |
-    |""".stripMargin
-    val conf = ConfigFactory.parseString(confStr)
-    val allConfs = conf +: config
-    BitcoinSAppConfig(allConfs: _*)
   }
 
   sealed trait ProjectType

testkit/src/main/scala/org/bitcoins/testkit/chain/ChainUnitTest.scala

@@ -184,7 +184,7 @@ trait ChainUnitTest
   def createPopulatedChainHandler(): Future[ChainHandler] = {
     for {
       blockHeaderDAO <- ChainUnitTest.createPopulatedBlockHeaderDAO()
-    } yield ChainHandler(blockHeaderDAO = blockHeaderDAO, appConfig)
+    } yield ChainHandler(blockHeaderDAO = blockHeaderDAO)
   }
 
   def withPopulatedChainHandler(test: OneArgAsyncTest): FutureOutcome = {
@@ -415,7 +415,7 @@ object ChainUnitTest extends BitcoinSLogger {
       ec: ExecutionContext): ChainHandler = {
     lazy val blockHeaderDAO = BlockHeaderDAO()
 
-    ChainHandler(blockHeaderDAO = blockHeaderDAO, appConfig)
+    ChainHandler(blockHeaderDAO)
   }
 
 }

testkit/src/main/scala/org/bitcoins/testkit/node/NodeUnitTest.scala

@@ -30,6 +30,9 @@ import org.scalatest.{
 import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
 import org.bitcoins.testkit.BitcoinSTestAppConfig
+import org.bitcoins.chain.blockchain.ChainHandler
+import org.bitcoins.chain.models.BlockHeaderDAO
+import org.bitcoins.node.SpvNodeCallbacks
 
 trait NodeUnitTest
     extends BitcoinSFixture
@@ -67,8 +70,11 @@ trait NodeUnitTest
   lazy val bitcoindPeerF = startedBitcoindF.map(NodeTestUtil.getBitcoindPeer)
 
   def buildPeerMessageReceiver(): PeerMessageReceiver = {
+
+    val dao = BlockHeaderDAO()
+    val chainHandler = ChainHandler(dao)
     val receiver =
-      PeerMessageReceiver.newReceiver()
+      PeerMessageReceiver.newReceiver(chainHandler, SpvNodeCallbacks.empty)
     receiver
   }
 

wallet-test/src/test/scala/org/bitcoins/wallet/WalletAppConfigTest.scala

@@ -10,9 +10,19 @@ import org.bitcoins.core.config.MainNet
 import org.bitcoins.wallet.config.WalletAppConfig
 import java.nio.file.Paths
 import org.bitcoins.core.hd.HDPurposes
+import java.nio.file.Files
+import ch.qos.logback.classic.Level
+import java.nio.file.Path
+import scala.util.Properties
 
 class WalletAppConfigTest extends BitcoinSUnitTest {
-  val config = WalletAppConfig()
+
+  val tempDir = Files.createTempDirectory("bitcoin-s")
+  val config = WalletAppConfig(directory = tempDir)
+
+  it must "resolve DB connections correctly " in {
+    assert(config.dbPath.startsWith(Properties.tmpDir))
+  }
 
   it must "be overridable" in {
     assert(config.network == RegTest)
@@ -27,27 +37,25 @@ class WalletAppConfigTest extends BitcoinSUnitTest {
   }
 
   it should "not matter how the overrides are passed in" in {
-    val dir = Paths.get("/", "bar", "biz")
     val overrider = ConfigFactory.parseString(s"""
     |bitcoin-s {
-    |  datadir = $dir 
     |  network = mainnet
     |}
     |""".stripMargin)
 
-    val throughConstuctor = WalletAppConfig(overrider)
+    val throughConstuctor = WalletAppConfig(tempDir, overrider)
     val throughWithOverrides = config.withOverrides(overrider)
     assert(throughWithOverrides.network == MainNet)
     assert(throughWithOverrides.network == throughConstuctor.network)
 
-    assert(throughWithOverrides.datadir.startsWith(dir))
     assert(throughWithOverrides.datadir == throughConstuctor.datadir)
 
   }
 
   it must "be overridable without screwing up other options" in {
-    val dir = Paths.get("/", "foo", "bar")
+    val otherConf = ConfigFactory.parseString(
-    val otherConf = ConfigFactory.parseString(s"bitcoin-s.datadir = $dir")
+      s"bitcoin-s.wallet.defaultAccountType = segwit"
+    )
     val thirdConf = ConfigFactory.parseString(
       s"bitcoin-s.wallet.defaultAccountType = nested-segwit")
 
@@ -55,9 +63,11 @@ class WalletAppConfigTest extends BitcoinSUnitTest {
 
     val twiceOverriden = overriden.withOverrides(thirdConf)
 
-    assert(overriden.datadir.startsWith(dir))
+    assert(overriden.defaultAccountKind == HDPurposes.SegWit)
-    assert(twiceOverriden.datadir.startsWith(dir))
     assert(twiceOverriden.defaultAccountKind == HDPurposes.NestedSegWit)
+
+    assert(config.datadir == overriden.datadir)
+    assert(twiceOverriden.datadir == overriden.datadir)
   }
 
   it must "be overridable with multiple levels" in {
@@ -65,6 +75,30 @@ class WalletAppConfigTest extends BitcoinSUnitTest {
     val mainnet = ConfigFactory.parseString("bitcoin-s.network = mainnet")
     val overriden: WalletAppConfig = config.withOverrides(testnet, mainnet)
     assert(overriden.network == MainNet)
+  }
 
+  it must "have user data directory configuration take precedence" in {
+
+    val tempDir = Files.createTempDirectory("bitcoin-s")
+    val tempFile = Files.createFile(tempDir.resolve("bitcoin-s.conf"))
+    val confStr = """
+    | bitcoin-s {
+    |   network = testnet3
+    |   
+    |   logging {
+    |     level = off
+    |
+    |     p2p = warn
+    |   }
+    | }
+    """.stripMargin
+    val _ = Files.write(tempFile, confStr.getBytes())
+
+    val appConfig = WalletAppConfig(directory = tempDir)
+
+    assert(appConfig.datadir == tempDir.resolve("testnet3"))
+    assert(appConfig.network == TestNet3)
+    assert(appConfig.logLevel == Level.OFF)
+    assert(appConfig.p2pLogLevel == Level.WARN)
   }
 }

wallet/src/main/scala/org/bitcoins/wallet/EncryptedMnemonic.scala

@@ -6,8 +6,7 @@ import scodec.bits.ByteVector
 
 import scala.util.{Failure, Success, Try}
 
-case class EncryptedMnemonic(value: AesEncryptedData, salt: AesSalt)
+case class EncryptedMnemonic(value: AesEncryptedData, salt: AesSalt) {
-    extends BitcoinSLogger {
 
   def toMnemonic(password: AesPassword): Try[MnemonicCode] = {
     import org.bitcoins.core.util.EitherUtil.EitherOps._

wallet/src/main/scala/org/bitcoins/wallet/Wallet.scala

@@ -6,7 +6,7 @@ import org.bitcoins.core.currency._
 import org.bitcoins.core.hd._
 import org.bitcoins.core.protocol.BitcoinAddress
 import org.bitcoins.core.protocol.transaction._
-import org.bitcoins.core.util.{BitcoinSLogger, EitherUtil}
+import org.bitcoins.core.util.EitherUtil
 import org.bitcoins.core.wallet.builder.BitcoinTxBuilder
 import org.bitcoins.core.wallet.fee.FeeUnit
 import org.bitcoins.core.wallet.utxo.BitcoinUTXOSpendingInfo
@@ -17,11 +17,9 @@ import scodec.bits.BitVector
 
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success, Try}
+import org.bitcoins.db.KeyHandlingLogger
 
-sealed abstract class Wallet
+sealed abstract class Wallet extends LockedWallet with UnlockedWalletApi {
-    extends LockedWallet
-    with UnlockedWalletApi
-    with BitcoinSLogger {
 
   /**
     * @inheritdoc
@@ -140,7 +138,7 @@ sealed abstract class Wallet
 }
 
 // todo: create multiple wallets, need to maintain multiple databases
-object Wallet extends CreateWalletApi with BitcoinSLogger {
+object Wallet extends CreateWalletApi with KeyHandlingLogger {
 
   private case class WalletImpl(
       mnemonicCode: MnemonicCode
@@ -235,6 +233,7 @@ object Wallet extends CreateWalletApi with BitcoinSLogger {
   private def createRootAccount(wallet: Wallet, purpose: HDPurpose)(
       implicit config: WalletAppConfig,
       ec: ExecutionContext): Future[AccountDb] = {
+
     val coin =
       HDCoin(purpose, HDUtil.getCoinType(config.network))
     val account = HDAccount(coin, 0)

wallet/src/main/scala/org/bitcoins/wallet/WalletStorage.scala

@@ -1,7 +1,6 @@
 package org.bitcoins.wallet
 
 import scala.collection.JavaConverters._
-import org.bitcoins.core.util.BitcoinSLogger
 import org.bitcoins.core.crypto.AesPassword
 import java.nio.file.Files
 import org.bitcoins.core.crypto.MnemonicCode
@@ -15,9 +14,10 @@ import java.nio.file.Path
 import scala.util.Try
 import org.bitcoins.wallet.config.WalletAppConfig
 import org.bitcoins.core.crypto.AesIV
+import org.bitcoins.db.KeyHandlingLogger
 
 // what do we do if seed exists? error if they aren't equal?
-object WalletStorage extends BitcoinSLogger {
+object WalletStorage extends KeyHandlingLogger {
 
   /** Checks if a wallet seed exists in datadir */
   def seedExists()(implicit config: WalletAppConfig): Boolean = {
@@ -183,6 +183,7 @@ object WalletStorage extends BitcoinSLogger {
   def decryptMnemonicFromDisk(passphrase: AesPassword)(
       implicit
       config: WalletAppConfig): ReadMnemonicResult = {
+
     val encryptedEither = readEncryptedMnemonicFromDisk()
 
     import org.bitcoins.core.util.EitherUtil.EitherOps._

wallet/src/main/scala/org/bitcoins/wallet/config/WalletAppConfig.scala

@@ -10,13 +10,24 @@ import java.nio.file.Files
 import org.bitcoins.core.hd.HDPurpose
 import org.bitcoins.core.hd.HDPurposes
 import org.bitcoins.core.hd.AddressType
+import java.nio.file.Path
 
-case class WalletAppConfig(private val conf: Config*) extends AppConfig {
+/** Configuration for the Bitcoin-S wallet
-  override val configOverrides: List[Config] = conf.toList
+  * @param directory The data directory of the wallet
-  override def moduleName: String = "wallet"
+  * @param confs Optional sequence of configuration overrides
-  override type ConfigType = WalletAppConfig
+  */
-  override def newConfigOfType(configs: Seq[Config]): WalletAppConfig =
+case class WalletAppConfig(
-    WalletAppConfig(configs: _*)
+    private val directory: Path,
+    private val conf: Config*)
+    extends AppConfig {
+  override protected[bitcoins] def configOverrides: List[Config] = conf.toList
+  override protected[bitcoins] def moduleName: String = "wallet"
+  override protected[bitcoins] type ConfigType = WalletAppConfig
+  override protected[bitcoins] def newConfigOfType(
+      configs: Seq[Config]): WalletAppConfig =
+    WalletAppConfig(directory, configs: _*)
+
+  protected[bitcoins] def baseDatadir: Path = directory
 
   lazy val defaultAccountKind: HDPurpose =
     config.getString("wallet.defaultAccountType") match {
@@ -63,3 +74,12 @@ case class WalletAppConfig(private val conf: Config*) extends AppConfig {
   }
 
 }
+
+object WalletAppConfig {
+
+  /** Constructs a wallet configuration from the default Bitcoin-S
+    * data directory and given list of configuration overrides.
+    */
+  def fromDefaultDatadir(confs: Config*): WalletAppConfig =
+    WalletAppConfig(AppConfig.DEFAULT_BITCOIN_S_DATADIR, confs: _*)
+}

wallet/src/main/scala/org/bitcoins/wallet/internal/AddressHandling.scala

@@ -22,12 +22,14 @@ import org.bitcoins.core.protocol.script.ScriptPubKey
 import org.bitcoins.core.protocol.transaction.TransactionOutPoint
 import org.bitcoins.core.number.UInt32
 import org.bitcoins.core.hd.AddressType
+import org.bitcoins.db.KeyHandlingLogger
 
 /**
   * Provides functionality related to addresses. This includes
   * enumeratng and creating them, primarily.
   */
-private[wallet] trait AddressHandling { self: LockedWallet =>
+private[wallet] trait AddressHandling extends KeyHandlingLogger {
+  self: LockedWallet =>
 
   override def listAddresses(): Future[Vector[AddressDb]] =
     addressDAO.findAll()

wallet/src/main/scala/org/bitcoins/wallet/internal/TransactionProcessing.scala

@@ -9,13 +9,15 @@ import org.bitcoins.wallet.api.AddUtxoSuccess
 import org.bitcoins.wallet.api.AddUtxoError
 import org.bitcoins.core.number.UInt32
 import org.bitcoins.core.util.FutureUtil
+import org.bitcoins.db.KeyHandlingLogger
 
 /** Provides functionality for processing transactions. This
   * includes importing UTXOs spent to our wallet, updating
   * confirmation counts and marking UTXOs as spent when
   * spending from our wallet
   */
-private[wallet] trait TransactionProcessing { self: LockedWallet =>
+private[wallet] trait TransactionProcessing extends KeyHandlingLogger {
+  self: LockedWallet =>
   /////////////////////
   // Public facing API
 

wallet/src/main/scala/org/bitcoins/wallet/internal/UtxoHandling.scala

@@ -22,6 +22,7 @@ import org.bitcoins.core.protocol.BitcoinAddress
 import scala.util.Success
 import scala.util.Failure
 import org.bitcoins.core.crypto.DoubleSha256DigestBE
+import org.bitcoins.db.KeyHandlingLogger
 
 /**
   * Provides functionality related to handling UTXOs in our wallet.
@@ -29,7 +30,8 @@ import org.bitcoins.core.crypto.DoubleSha256DigestBE
   * UTXOs in the wallet and importing a UTXO into the wallet for later
   * spending.
   */
-private[wallet] trait UtxoHandling { self: LockedWallet =>
+private[wallet] trait UtxoHandling extends KeyHandlingLogger {
+  self: LockedWallet =>
 
   /** @inheritdoc */
   override def listUtxos(): Future[Vector[SpendingInfoDb]] =

wallet/src/main/scala/org/bitcoins/wallet/models/AddressDAO.scala

@@ -15,8 +15,8 @@ import org.bitcoins.core.protocol.script.ScriptPubKey
 import org.bitcoins.core.hd.HDPurpose
 
 case class AddressDAO()(
-    implicit val ec: ExecutionContext,
+    implicit ec: ExecutionContext,
-    val appConfig: WalletAppConfig
+    config: WalletAppConfig
 ) extends CRUD[AddressDb, BitcoinAddress] {
   import org.bitcoins.db.DbCommonsColumnMappers._
 

wallet/src/main/scala/org/bitcoins/wallet/models/SpendingInfoTable.scala

@@ -16,7 +16,6 @@ import org.bitcoins.core.hd.HDPath
 
 import org.bitcoins.core.hd.SegWitHDPath
 import org.bitcoins.core.crypto.BIP39Seed
-import org.bitcoins.core.util.BitcoinSLogger
 import org.bitcoins.core.hd.LegacyHDPath
 import org.bitcoins.core.crypto.DoubleSha256DigestBE
 
@@ -85,9 +84,7 @@ case class LegacySpendingInfo(
   * we need to derive the private keys, given
   * the root wallet seed.
   */
-sealed trait SpendingInfoDb
+sealed trait SpendingInfoDb extends DbRowAutoInc[SpendingInfoDb] {
-    extends DbRowAutoInc[SpendingInfoDb]
-    with BitcoinSLogger {
 
   protected type PathType <: HDPath
 
@@ -141,13 +138,6 @@ sealed trait SpendingInfoDb
 
     val sign: Sign = Sign(privKey.signFunction, pubAtPath)
 
-    logger.info({
-      val shortStr = s"${outPoint.txId.hex}:${outPoint.vout.toInt}"
-      val detailsStr =
-        s"scriptPubKey=${output.scriptPubKey}, amount=${output.value}, keyPath=${privKeyPath}, pubKey=${pubAtPath}"
-      s"Converting DB UTXO $shortStr ($detailsStr) to spending info"
-    })
-
     BitcoinUTXOSpendingInfo(outPoint,
                             output,
                             List(sign),