Build and CI improvements (#710)

* Build and CI improvements

In this commit we:

1) Parallelize the Travis CI config, by splitting
    each project into its own Travis task
2) Download bitcoind binaries through sbt

* Use binaries downloaded by sbt task

* Make BitcoindRpcTestUtil work on Travis without bitcoind on PATH

* Add new downloadEclair task to sbt

* use sbt downloaded binaries in tests

* Fix Eclair and Bitcoind tests

.gitignore

@@ -1,6 +1,10 @@
 *.class
 *.log
 
+# binaries downloaded by sbt for tests
+binaries/bitcoind
+binaries/eclair
+
 # sbt specific
 .cache
 .history
@@ -72,4 +76,4 @@ libsecp256k1.pc
 # Docusaurs
 node_modules
 website/build
-website/static/api
+website/static/api

.travis.yml

@@ -1,52 +1,89 @@
-# We've been seeing very strange errors
-# where bitcoind returns a 503 error.
-# We have a suspicion that this comes from
-# a service running on the Travis servers
-# in non-sudo mode
-sudo: true
-
 language: scala
 
-#https://docs.travis-ci.com/user/reference/osx#jdk-and-os-x
-#Note: osx uses jdk10 by default which is NOT officially supported by scala
-#This does not seem to be causing any errors right now
-os:
-  - linux
-  - osx
+env:
+  matrix:
+    - TEST_COMMAND="bitcoindRpcTest/test bitcoindRpc/coverageReport bitcoindRpc/coverageAggregate bitcoindRpc/coveralls"
+    - TEST_COMMAND="chainTest/test chain/coverageReport chain/coverageAggregate chain/coveralls"
+    - TEST_COMMAND="eclairRpcTest/test eclairRpc/coverageReport eclairRpc/coverageAggregate eclairRpc/coveralls"
+    - TEST_COMMAND="walletTest/test wallet/coverageReport wallet/coverageAggregate wallet/coveralls nodeTest/test node/coverageReport node/coverageAggregate node/coveralls"
+    - TEST_COMMAND="coreTest/test core/coverageReport core/coverageAggregate core/coveralls secp256k1jni/test zmq/test zmq/coverageReport zmq/coverageAggregate zmq/coveralls"
 
+os: linux
 scala:
-  - 2.11.12
   - 2.12.9
   - 2.13.0
 
+# Fiddling with Travis config is not fun:-(
+# To avoid spending too much time waiting on Travis, you
+# can use this tool to parse the config file locally: https://github.com/travis-ci/travis-yml
+# After getting it set up, do:
+# $ curl -X POST --data-binary @.travis.yml localhost:9292/v1/parse | jq
+# this should return a big JSON object, where especially
+# config.matrix.include tells you a lot about what the build
+# is going to look like
+matrix:
+  include:
+    # this way of including jobs is not ideal... unfortunately it's not
+    # possible to nest env.matrix. could a better solution be to write
+    # a small script that generates a Travis config for us?
+    - os: linux
+      name: "Linux compile for 2.11"
+      env:
+        - TEST_COMMAND="test:compile"
+      scala:
+        - 2.11.12
+    - os: osx
+      name: "macOS bitcoind tests"
+      env:
+        - TEST_COMMAND="bitcoindRpcTest/test bitcoindRpc/coverageReport bitcoindRpc/coverageAggregate bitcoindRpc/coveralls"
+      scala:
+        - 2.13.0
+    - os: osx
+      name: "macOS Eclair tests"
+      env:
+        - TEST_COMMAND="eclairRpcTest/test eclairRpc/coverageReport eclairRpc/coverageAggregate eclairRpc/coveralls"
+      scala:
+        - 2.13.0
+    - os: osx
+      name: "macOS node and wallet tests"
+      env:
+        - TEST_COMMAND="walletTest/test wallet/coverageReport wallet/coverageAggregate wallet/coveralls nodeTest/test node/coverageReport node/coverageAggregate node/coveralls"
+      scala:
+        - 2.13.0
+
+    # compile website, to check for documentation regressions
+    - stage: test
+      name: Compile website
+      script: sbt docs/mdoc
+
+    # Release snapshots/versions of all libraries
+    # run ci-release only if previous stages passed
+    - stage: release
+      jdk: openjdk8
+      name: Publish library
+      script: sbt ci-release
+
+    # run website push only if website compilation passed
+    # we use custom sbt task that first compiles Scaladocs
+    # and then calls the docusaurusPublishGhpages task
+    - script: sbt docs/publishWebsite
+      name: Publish website
+
 # These directories are cached to S3 at the end of the build
+# https://www.scala-sbt.org/1.x/docs/Travis-CI-with-sbt.html#Caching
 cache:
   directories:
     - $HOME/.ivy2/cache
     - $HOME/.sbt/boot/
+    - $PWD/binaries/bitcoind/
+    - $PWD/binaries/eclair/
 
+# https://www.scala-sbt.org/1.x/docs/Travis-CI-with-sbt.html#Caching
 before_cache:
-  # Tricks to avoid unnecessary cache updates
-  - find $HOME/.sbt -name "*.lock" | xargs rm
-  - find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm
-
-install:
-  - if [[ $TRAVIS_OS_NAME == "linux" ]]; then export PLATFORM="x86_64-linux-gnu"; else export PLATFORM="osx64"; fi
-  # # # bitcoind v16
-  - wget https://bitcoincore.org/bin/bitcoin-core-0.16.3/bitcoin-0.16.3-${PLATFORM}.tar.gz
-  - tar -xzf bitcoin-0.16.3-${PLATFORM}.tar.gz
-  - export BITCOIND_V16_PATH=$(pwd)/bitcoin-0.16.3/bin
-  # # # bitcoind v17
-  - wget https://bitcoincore.org/bin/bitcoin-core-0.17.0.1/bitcoin-0.17.0.1-${PLATFORM}.tar.gz
-  - tar -xzf bitcoin-0.17.0.1-${PLATFORM}.tar.gz
-  # tar places the unpacked directory in a not so intuitive location
-  - export BITCOIND_V17_PATH=$(pwd)/bitcoin-0.17.0/bin
-  # set default bitcoind to randomly choose between 0.16 and 0.17
-  - if [ $(($RANDOM%2)) == 1 ]; then BITCOIND_PATH=$BITCOIND_V16_PATH; else BITCOIND_PATH=$BITCOIND_V17_PATH; fi;
-  - export PATH=$BITCOIND_PATH:$PATH
-  # # # Eclair
-  - wget https://github.com/ACINQ/eclair/releases/download/v0.3.1/eclair-node-0.3.1-6906ecb.jar
-  - export ECLAIR_PATH=$(pwd)
+  # Cleanup the cached directories to avoid unnecessary cache updates
+  - rm -fv $HOME/.ivy2/.sbt.ivy.lock
+  - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete
+  - find $HOME/.sbt        -name "*.lock"               -print -delete
 
 before_script:
   - git fetch --tags
@@ -61,24 +98,7 @@ stages:
   - name: release
     if: ((branch = master AND type = push) OR (tag IS present)) AND NOT fork
 
-script: sbt ++$TRAVIS_SCALA_VERSION coverage test &&
-  sbt ++$TRAVIS_SCALA_VERSION core/coverageReport &&
-  sbt ++$TRAVIS_SCALA_VERSION chain/coverageReport &&
-  sbt ++$TRAVIS_SCALA_VERSION coverageAggregate &&
-  sbt ++$TRAVIS_SCALA_VERSION coveralls
-
-jobs:
-  include:
-    - stage: test
-      name: Compile website
-      script: sbt docs/mdoc
-    # run ci-release only if previous stages passed
-    - stage: release
-      jdk: openjdk8
-      name: Publish library
-      script: sbt ci-release
-    # run website push only if previous stages passed
-    # we use custom sbt task that first compiles Scaladocs
-    # and then calls the docusaurusPublishGhpages task
-    - script: sbt docs/publishWebsite
-      name: Publish website
+script:
+  # Modify PATH to include binaries we are about to download
+  - export PATH=$PWD/binaries/bitcoind/bitcoin-0.17.0/bin/:$PATH
+  - sbt ++$TRAVIS_SCALA_VERSION downloadBitcoind downloadEclair coverage $TEST_COMMAND

bitcoind-rpc-test/bitcoind-rpc-test.sbt

@@ -0,0 +1,12 @@
+name := "bitcoin-s-bitcoind-rpc-test"
+
+libraryDependencies ++= Deps.bitcoindRpcTest(scalaVersion.value)
+
+lazy val downloadBitcoind = taskKey[Unit] {
+  "Download bitcoind binaries, extract to ./bitcoind-binaries"
+}
+
+import java.nio.file.Paths
+lazy val bitcoindRpc = project in Paths.get("..", "bitcoind-rpc").toFile
+
+Test / test := (Test / test dependsOn bitcoindRpc / downloadBitcoind).value

bitcoind-rpc/README.md

@@ -1,21 +1,2 @@
-See the `bitcoind`/Bitcoin Core section on the 
+See the `bitcoind`/Bitcoin Core section on the
 Bitcoin-S [website](https://bitcoin-s.org/docs/rpc/rpc-bitcoind).
-
-## Testing
-
-To test the Bitcoin-S RPC project you need both version 0.16 and 0.17 of Bitcoin Core. A list of current and previous releases can be found [here](https://bitcoincore.org/en/releases/).
-
-You then need to set environment variables to indicate where Bitcoin-S can find the different versions: 
-
-```bash
-$ export BITCOIND_V16_PATH=/path/to/v16/bitcoind
-$ export BITCOIND_V17_PATH=/path/to/v17/bitcoind
-```
-
-If you just run tests testing common functionality it's enough to have either version 0.16 or 0.17 on your `PATH`.
-
-To run all RPC related tests:
-
-```bash
-$ bloop test bitcoindRpcTest
-```

bitcoind-rpc/bitcoind-rpc.sbt

@@ -0,0 +1,72 @@
+import scala.util.Properties
+import scala.collection.JavaConverters._
+import java.nio.file.Files
+import java.nio.file.Paths
+
+name := "bitcoin-s-bitcoind-rpc"
+
+libraryDependencies ++= Deps.bitcoindRpc
+
+dependsOn {
+  lazy val core = project in Paths.get("..", "core").toFile
+  core
+}
+
+lazy val downloadBitcoind = taskKey[Unit] {
+  "Download bitcoind binaries, extract to ./binaries/bitcoind"
+}
+
+downloadBitcoind := {
+  val logger = streams.value.log
+  import scala.sys.process._
+
+  val binaryDir = Paths.get("binaries", "bitcoind")
+
+  if (Files.notExists(binaryDir)) {
+    logger.info(s"Creating directory for bitcoind binaries: $binaryDir")
+    Files.createDirectories(binaryDir)
+  }
+
+  val versions = List("0.17.0.1", "0.16.3")
+
+  logger.debug(
+    s"(Maybe) downloading Bitcoin Core binaries for versions: ${versions.mkString(",")}")
+
+  val platform =
+    if (Properties.isLinux) "x86_64-linux-gnu"
+    else if (Properties.isMac) "osx64"
+    else sys.error(s"Unsupported OS: ${Properties.osName}")
+
+  versions.foreach { version =>
+    val versionDir = binaryDir resolve version
+    val archiveLocation = binaryDir resolve s"$version.tar.gz"
+    val location =
+      s"https://bitcoincore.org/bin/bitcoin-core-$version/bitcoin-$version-$platform.tar.gz"
+
+    val expectedEndLocation = binaryDir resolve s"bitcoin-$version"
+    if (Files
+          .list(binaryDir)
+          .iterator
+          .asScala
+          .map(_.toString)
+          .exists(expectedEndLocation.toString.startsWith(_))) {
+      logger.debug(
+        s"Directory $expectedEndLocation already exists, skipping download of version $version")
+    } else {
+      logger.info(
+        s"Downloading bitcoind version $version from location: $location")
+      logger.info(s"Placing the file in $archiveLocation")
+      val downloadCommand = url(location) #> archiveLocation.toFile
+      downloadCommand.!!
+
+      logger.info(s"Download complete, unzipping result")
+
+      val extractCommand = s"tar -xzf $archiveLocation --directory $binaryDir"
+      logger.info(s"Extracting archive with command: $extractCommand")
+      extractCommand.!!
+
+      logger.info(s"Deleting archive")
+      Files.delete(archiveLocation)
+    }
+  }
+}

bitcoind-rpc/src/main/scala/org/bitcoins/rpc/client/common/BitcoindRpcClient.scala

@@ -60,6 +60,9 @@ sealed trait BitcoindVersion
 
 object BitcoindVersion {
 
+  /** The newest `bitcoind` version supported by Bitcoin-S */
+  val newest = V17
+
   case object V16 extends BitcoindVersion {
     override def toString: String = "v0.16"
   }

build.sbt

@@ -382,15 +382,10 @@ lazy val zmq = project
 lazy val bitcoindRpc = project
   .in(file("bitcoind-rpc"))
   .settings(commonProdSettings: _*)
-  .settings(name := "bitcoin-s-bitcoind-rpc",
-            libraryDependencies ++= Deps.bitcoindRpc)
-  .dependsOn(core)
 
 lazy val bitcoindRpcTest = project
   .in(file("bitcoind-rpc-test"))
   .settings(commonTestSettings: _*)
-  .settings(libraryDependencies ++= Deps.bitcoindRpcTest(scalaVersion.value),
-            name := "bitcoin-s-bitcoind-rpc-test")
   .dependsOn(core % testAndCompile, testkit)
 
 lazy val bench = project
@@ -406,12 +401,6 @@ lazy val bench = project
 lazy val eclairRpc = project
   .in(file("eclair-rpc"))
   .settings(commonProdSettings: _*)
-  .settings(name := "bitcoin-s-eclair-rpc",
-            libraryDependencies ++= Deps.eclairRpc)
-  .dependsOn(
-    core,
-    bitcoindRpc
-  )
 
 lazy val eclairRpcTest = project
   .in(file("eclair-rpc-test"))

docs/rpc/bitcoind.md

@@ -124,22 +124,3 @@ val txid: Future[DoubleSha256DigestBE] =
       }
   }
 ```
-
-## Testing
-
-To test the Bitcoin-S RPC project you need both version 0.16 and 0.17 of Bitcoin Core. A list of current and previous releases can be found [here](https://bitcoincore.org/en/releases/).
-
-You then need to set environment variables to indicate where Bitcoin-S can find the different versions:
-
-```bash
-$ export BITCOIND_V16_PATH=/path/to/v16/bitcoind
-$ export BITCOIND_V17_PATH=/path/to/v17/bitcoind
-```
-
-If you just run tests testing common functionality it's enough to have either version 0.16 or 0.17 on your `PATH`.
-
-To run all RPC related tests:
-
-```bash
-$ bash sbt bitcoindRpcTest/test
-```

docs/rpc/eclair.md

@@ -22,6 +22,7 @@ To run Eclair you can use this command:
 $ java -jar eclair-node-0.2-beta8-52821b8.jar &
 ```
 
-Alternatively you can set the `ECLAIR_PATH` env variable and then you can start Eclair with the `start` method on `EclairRpcClient`.
+If you wish to start Eclair from the RPC client, you can do one of the following:
 
-**YOU NEED TO SET `ECLAIR_PATH` CORRECTLY TO BE ABLE TO RUN THE UNIT TESTS**
+1. Construct a `EclairRpcClient` with the `binary` field set
+2. Set the `ECLAIR_PATH` environment variable to the directory where the Eclair Jar is located.

eclair-rpc-test/eclair-rpc-test.sbt

@@ -0,0 +1,8 @@
+lazy val downloadEclair = taskKey[Unit] {
+  "Download Eclair binaries, extract ./binaries/eclair"
+}
+
+import java.nio.file.Paths
+lazy val eclairRpc = project in Paths.get("..", "eclair-rpc").toFile
+
+Test / test := (Test / test dependsOn eclairRpc / downloadEclair).value

eclair-rpc-test/src/test/scala/org/bitcoins/eclair/rpc/EclairRpcClientTest.scala

@@ -37,9 +37,28 @@ import org.bitcoins.core.protocol.ln.{
 import org.bitcoins.testkit.async.TestAsyncUtil
 
 import scala.concurrent.duration._
+import java.nio.file.Files
 
 class EclairRpcClientTest extends AsyncFlatSpec with BeforeAndAfterAll {
 
+  private val dirExists = Files.exists(EclairRpcTestUtil.binaryDirectory)
+  private val hasContents = dirExists && Files
+    .list(EclairRpcTestUtil.binaryDirectory)
+    .toArray()
+    .nonEmpty
+
+  if (!hasContents) {
+    import System.err.{println => printerr}
+    printerr()
+    printerr(s"Run 'sbt downloadEclair' to fetch needed binaries")
+    sys.error {
+      val msg =
+        s""""Eclair binary directory (${BitcoindRpcTestUtil.binaryDirectory}) is empty. 
+        |Run 'sbt downloadEclair' to fetch needed binaries""".stripMargin
+      msg
+    }
+  }
+
   implicit val system: ActorSystem =
     ActorSystem("EclairRpcClient", BitcoindRpcTestUtil.AKKA_CONFIG)
   implicit val m: ActorMaterializer = ActorMaterializer.create(system)
@@ -326,7 +345,7 @@ class EclairRpcClientTest extends AsyncFlatSpec with BeforeAndAfterAll {
       bitcoind <- EclairRpcTestUtil.startedBitcoindRpcClient()
       eclair <- {
         val server = EclairRpcTestUtil.eclairInstance(bitcoind)
-        val eclair = new EclairRpcClient(server)
+        val eclair = new EclairRpcClient(server, EclairRpcTestUtil.binary)
         eclair.start().map(_ => eclair)
       }
       _ <- TestAsyncUtil.retryUntilSatisfiedF(conditionF =
@@ -451,7 +470,8 @@ class EclairRpcClientTest extends AsyncFlatSpec with BeforeAndAfterAll {
       executeWithClientOtherClient(getBadInstance)
     }
 
-    val badClientF = badInstanceF.map(new EclairRpcClient(_))
+    val badClientF =
+      badInstanceF.map(new EclairRpcClient(_, EclairRpcTestUtil.binary))
 
     badClientF.flatMap { badClient =>
       recoverToSucceededIf[RuntimeException](badClient.getInfo)

eclair-rpc/eclair-rpc.sbt

@@ -0,0 +1,50 @@
+import java.nio.file._
+
+name := "bitcoin-s-eclair-rpc"
+
+libraryDependencies ++= Deps.eclairRpc
+
+dependsOn {
+  lazy val bitcoindRpc = project in Paths.get("..", "bitcoind-rpc").toFile
+  bitcoindRpc
+}
+
+lazy val downloadEclair = taskKey[Unit] {
+  "Download Eclair binaries, extract ./binaries/eclair"
+}
+
+downloadEclair := {
+  val logger = streams.value.log
+  import scala.sys.process._
+
+  val binaryDir = Paths.get("binaries", "eclair")
+
+  if (Files.notExists(binaryDir)) {
+    logger.info(s"Creating directory for Eclair binaires: $binaryDir")
+    Files.createDirectories(binaryDir)
+  }
+
+  val version = "0.3.1"
+  val commit = "6906ecb"
+
+  logger.debug(s"(Maybe) downloading Eclair binaries for version: $version")
+
+  val versionDir = binaryDir resolve version
+  val location =
+    s"https://github.com/ACINQ/eclair/releases/download/v$version/eclair-node-$version-$commit.jar"
+
+  if (Files.exists(versionDir)) {
+    logger.debug(
+      s"Directory $versionDir already exists, skipping download of Eclair $version")
+  } else {
+    logger.info(s"Creating directory $version")
+    Files.createDirectories(versionDir)
+
+    val destination = versionDir resolve s"eclair-node-$version-$commit.jar"
+    logger.info(
+      s"Downloading Eclair $version from location: $location, to destination: $destination")
+    (url(location) #> destination.toFile).!!
+
+    logger.info(s"Download complete")
+  }
+}

eclair-rpc/src/main/scala/org/bitcoins/eclair/rpc/client/EclairRpcClient.scala

@@ -37,8 +37,13 @@ import scala.concurrent.duration.{DurationInt, FiniteDuration}
 import scala.concurrent.{ExecutionContext, Future, Promise}
 import scala.sys.process._
 import scala.util.{Failure, Properties, Success}
+import java.nio.file.NoSuchFileException
 
-class EclairRpcClient(val instance: EclairInstance)(
+/**
+  * @param binary Path to Eclair Jar. If not present, reads
+  *               environment variable `ECLAIR_PATH`
+  */
+class EclairRpcClient(val instance: EclairInstance, binary: Option[File] = None)(
     implicit system: ActorSystem)
     extends EclairApi {
   import JsonReaders._
@@ -633,21 +638,33 @@ class EclairRpcClient(val instance: EclairInstance)(
   }
 
   private def pathToEclairJar: String = {
-    val path = Properties
-      .envOrNone("ECLAIR_PATH")
-      .getOrElse(throw new RuntimeException(
-        List("Environment variable ECLAIR_PATH is not set!",
-             "This needs to be set to the directory containing the Eclair Jar")
-          .mkString(" ")))
 
-    val eclairV = "/eclair-node-0.3.1-6906ecb.jar"
-    val fullPath = path + eclairV
+    (binary, Properties.envOrNone("ECLAIR_PATH")) match {
+      // default to provided binary
+      case (Some(binary), _) =>
+        if (binary.exists) {
+          binary.toString
+        } else {
+          throw new NoSuchFileException(
+            s"Given binary ($binary) does not exist!")
+        }
+      case (None, Some(path)) =>
+        val eclairV =
+          s"/eclair-node-${EclairRpcClient.version}-${EclairRpcClient.commit}.jar"
+        val fullPath = path + eclairV
 
-    val jar = new File(fullPath)
-    if (jar.exists) {
-      fullPath
-    } else {
-      throw new RuntimeException(s"Could not Eclair Jar at location $fullPath")
+        val jar = new File(fullPath)
+        if (jar.exists) {
+          fullPath
+        } else {
+          throw new NoSuchFileException(
+            s"Could not Eclair Jar at location $fullPath")
+        }
+      case (None, None) =>
+        val msg = List(
+          "Environment variable ECLAIR_PATH is not set, and no binary is given!",
+          "Either needs to be set in order to start Eclair.")
+        throw new RuntimeException(msg.mkString(" "))
     }
   }
 
@@ -774,3 +791,12 @@ class EclairRpcClient(val instance: EclairInstance)(
     f
   }
 }
+
+object EclairRpcClient {
+
+  /** The current commit we support of Eclair */
+  private[bitcoins] val commit = "6906ecb"
+
+  /** The current version we support of Eclair */
+  private[bitcoins] val version = "0.3.1"
+}

testkit/src/main/scala/org/bitcoins/testkit/eclair/rpc/EclairRpcTestUtil.scala

@@ -29,6 +29,9 @@ import scala.concurrent.duration.{DurationInt, FiniteDuration}
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success}
 import org.bitcoins.rpc.config.BitcoindAuthCredentials
+import java.nio.file.Paths
+import scala.util.Properties
+import java.nio.file.Files
 
 /**
   * @define nodeLinkDoc
@@ -44,6 +47,32 @@ import org.bitcoins.rpc.config.BitcoindAuthCredentials
 trait EclairRpcTestUtil extends BitcoinSLogger {
   import org.bitcoins.core.compat.JavaConverters._
 
+  /** Directory where sbt downloads Eclair binaries */
+  private[bitcoins] lazy val binaryDirectory = {
+    val baseDirectory = {
+      val cwd = Paths.get(Properties.userDir)
+      if (cwd.endsWith("eclair-rpc-test") || cwd.endsWith("bitcoind-rpc-test")) {
+        cwd.getParent()
+      } else cwd
+    }
+
+    baseDirectory.resolve("binaries").resolve("eclair")
+  }
+
+  /** Path to Jar downloaded by Eclair, if it exists */
+  private[bitcoins] lazy val binary: Option[File] = {
+    val path = binaryDirectory
+      .resolve(EclairRpcClient.version)
+      .resolve(
+        s"eclair-node-${EclairRpcClient.version}-${EclairRpcClient.commit}.jar")
+
+    if (Files.exists(path)) {
+      Some(path.toFile)
+    } else {
+      None
+    }
+  }
+
   def randomDirName: String =
     0.until(5).map(_ => scala.util.Random.alphanumeric.head).mkString
 
@@ -169,7 +198,7 @@ trait EclairRpcTestUtil extends BitcoinSLogger {
     }
 
     val randInstanceF = bitcoindRpcF.map(randomEclairInstance(_))
-    val eclairRpcF = randInstanceF.map(i => new EclairRpcClient(i))
+    val eclairRpcF = randInstanceF.map(i => new EclairRpcClient(i, binary))
 
     val startedF = eclairRpcF.flatMap(_.start())
 
@@ -179,7 +208,7 @@ trait EclairRpcTestUtil extends BitcoinSLogger {
   def cannonicalEclairClient()(
       implicit system: ActorSystem): EclairRpcClient = {
     val inst = cannonicalEclairInstance()
-    new EclairRpcClient(inst)
+    new EclairRpcClient(inst, binary)
   }
 
   def deleteTmpDir(dir: File): Boolean = {
@@ -441,13 +470,13 @@ trait EclairRpcTestUtil extends BitcoinSLogger {
       bitcoindRpcClientF.map(EclairRpcTestUtil.eclairInstance(_))
 
     val clientF = e1InstanceF.flatMap { e1 =>
-      val e = new EclairRpcClient(e1)
+      val e = new EclairRpcClient(e1, binary)
       logger.debug(
         s"Temp eclair directory created ${e.getDaemon.authCredentials.datadir}")
       e.start().map(_ => e)
     }
     val otherClientF = e2InstanceF.flatMap { e2 =>
-      val e = new EclairRpcClient(e2)
+      val e = new EclairRpcClient(e2, binary)
       logger.debug(
         s"Temp eclair directory created ${e.getDaemon.authCredentials.datadir}")
       e.start().map(_ => e)

testkit/src/main/scala/org/bitcoins/testkit/rpc/BitcoindRpcTestUtil.scala

@@ -46,6 +46,7 @@ import org.bitcoins.util.ListUtil
 import scala.annotation.tailrec
 import scala.collection.immutable.Map
 import scala.collection.mutable
+import scala.collection.JavaConverters._
 import scala.concurrent._
 import scala.concurrent.duration.{DurationInt, FiniteDuration}
 import scala.util._
@@ -54,6 +55,10 @@ import java.io.File
 import com.typesafe.config.Config
 import com.typesafe.config.ConfigFactory
 import java.nio.file.Path
+import org.bitcoins.rpc.client.common.BitcoindVersion.Unknown
+import org.bitcoins.rpc.client.common.BitcoindVersion.V16
+import org.bitcoins.rpc.client.common.BitcoindVersion.V17
+import java.nio.file.Files
 
 //noinspection AccessorLikeMethodIsEmptyParen
 trait BitcoindRpcTestUtil extends BitcoinSLogger {
@@ -141,33 +146,46 @@ trait BitcoindRpcTestUtil extends BitcoinSLogger {
 
   lazy val network: RegTest.type = RegTest
 
-  private val V16_ENV = "BITCOIND_V16_PATH"
-  private val V17_ENV = "BITCOIND_V17_PATH"
-
-  private def getFileFromEnv(env: String): File = {
-    val envValue = Properties
-      .envOrNone(env)
-      .getOrElse(
-        throw new IllegalArgumentException(
-          s"$env environment variable is not set"))
-
-    val maybeDir = new File(envValue.trim)
-
-    val binary = if (maybeDir.isDirectory) {
-      Paths.get(maybeDir.getAbsolutePath, "bitcoind").toFile
-    } else {
-      maybeDir
+  /** The directory that sbt downloads bitcoind binaries into */
+  private[bitcoins] val binaryDirectory = {
+    val baseDirectory = {
+      val cwd = Paths.get(Properties.userDir)
+      if (cwd.endsWith("bitcoind-rpc-test") || cwd.endsWith("eclair-rpc-test")) {
+        cwd.getParent()
+      } else cwd
     }
 
-    binary
+    baseDirectory.resolve("binaries").resolve("bitcoind")
   }
 
-  private def getBinary(version: BitcoindVersion): File =
-    version match {
-      case BitcoindVersion.V16     => getFileFromEnv(V16_ENV)
-      case BitcoindVersion.V17     => getFileFromEnv(V17_ENV)
-      case BitcoindVersion.Unknown => BitcoindInstance.DEFAULT_BITCOIND_LOCATION
-    }
+  private def getBinary(version: BitcoindVersion): File = version match {
+    // default to newest version
+    case Unknown => getBinary(BitcoindVersion.newest)
+    case known @ (V16 | V17) =>
+      val versionFolder = Files
+        .list(binaryDirectory)
+        .iterator()
+        .asScala
+        .toList
+        .filter { f =>
+          val isFolder = Files.isDirectory(f)
+          val matchesVersion = f.toString.contains {
+            // drop leading 'v'
+            known.toString.drop(1)
+          }
+          isFolder && matchesVersion
+        }
+        // might be multiple versions downloaded for
+        // each major version, i.e. 0.16.2 and 0.16.3
+        .sorted
+        // we want the most recent one
+        .last
+
+      versionFolder
+        .resolve("bin")
+        .resolve("bitcoind")
+        .toFile()
+  }
 
   /** Creates a `bitcoind` instance within the user temporary directory */
   def instance(
@@ -181,10 +199,26 @@ trait BitcoindRpcTestUtil extends BitcoinSLogger {
     val configFile = writtenConfig(uri, rpcUri, zmqPort, pruneMode)
     val conf = BitcoindConfig(configFile)
     val auth = BitcoindAuthCredentials.fromConfig(conf)
-    val binary = versionOpt match {
-      case Some(version) =>
-        getBinary(version)
-      case None => BitcoindInstance.DEFAULT_BITCOIND_LOCATION
+    val binary: File = versionOpt match {
+      case Some(version) => getBinary(version)
+      case None =>
+        Try {
+          BitcoindInstance.DEFAULT_BITCOIND_LOCATION
+        }.recoverWith {
+          case _: RuntimeException =>
+            if (Files.exists(
+                  BitcoindRpcTestUtil.binaryDirectory
+                )) {
+              Success(getBinary(BitcoindVersion.newest))
+            } else {
+              Failure(new RuntimeException(
+                "Could not locate bitcoind. Make sure it is installed on your PATH, or if working with Bitcoin-S directly, try running 'sbt downloadBitcoind'"))
+            }
+
+        } match {
+          case Failure(exception) => throw exception
+          case Success(value)     => value
+        }
     }
     val instance = BitcoindInstance(network = network,
                                     uri = uri,

testkit/src/main/scala/org/bitcoins/testkit/util/BitcoindRpcTest.scala

@@ -10,8 +10,28 @@ import org.slf4j.{Logger, LoggerFactory}
 import scala.collection.mutable
 import scala.concurrent.duration.DurationInt
 import scala.concurrent.{Await, ExecutionContext}
+import java.nio.file.Files
 
 abstract class BitcoindRpcTest extends AsyncFlatSpec with BeforeAndAfterAll {
+
+  private val dirExists = Files.exists(BitcoindRpcTestUtil.binaryDirectory)
+  private val hasContents = dirExists && Files
+    .list(BitcoindRpcTestUtil.binaryDirectory)
+    .toArray()
+    .nonEmpty
+
+  if (!hasContents) {
+    import System.err.{println => printerr}
+    printerr()
+    printerr(s"Run 'sbt downloadBitcoind' to fetch needed binaries")
+    sys.error {
+      val msg =
+        s""""bitcoind binary directory (${BitcoindRpcTestUtil.binaryDirectory}) is empty. 
+        |Run 'sbt downloadBitcoind' to fetch needed binaries""".stripMargin
+      msg
+    }
+  }
+
   protected val logger: Logger = LoggerFactory.getLogger(getClass)
 
   implicit val system: ActorSystem =